diff --git "a/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/logs/debug-internal.log" "b/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/logs/debug-internal.log" @@ -0,0 +1,43270 @@ +2024-03-13 11:28:08,064 INFO StreamThr :4098812 [internal.py:wandb_internal():86] W&B internal server running at pid: 4098812, started at: 2024-03-13 11:28:08.062782 +2024-03-13 11:28:08,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status +2024-03-13 11:28:08,077 INFO WriterThread:4098812 [datastore.py:open_for_write():87] open: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/run-6hbull8s.wandb +2024-03-13 11:28:08,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: header +2024-03-13 11:28:08,095 DEBUG SenderThread:4098812 [sender.py:send():382] send: run +2024-03-13 11:28:08,611 INFO SenderThread:4098812 [dir_watcher.py:__init__():211] watching files in: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files +2024-03-13 11:28:08,612 INFO SenderThread:4098812 [sender.py:_start_run_threads():1136] run started: 6hbull8s with start time 1710329288.064789 +2024-03-13 11:28:08,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: check_version +2024-03-13 11:28:08,624 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: check_version +2024-03-13 11:28:08,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: run_start +2024-03-13 11:28:08,686 DEBUG HandlerThread:4098812 [system_info.py:__init__():27] System info init +2024-03-13 11:28:08,686 DEBUG HandlerThread:4098812 [system_info.py:__init__():42] System info init done +2024-03-13 11:28:08,686 INFO HandlerThread:4098812 [system_monitor.py:start():194] Starting system monitor +2024-03-13 11:28:08,686 INFO SystemMonitor:4098812 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-13 11:28:08,687 INFO HandlerThread:4098812 [system_monitor.py:probe():214] Collecting system info +2024-03-13 11:28:08,687 INFO SystemMonitor:4098812 [interfaces.py:start():190] Started cpu monitoring +2024-03-13 11:28:08,687 INFO SystemMonitor:4098812 [interfaces.py:start():190] Started disk monitoring +2024-03-13 11:28:08,688 INFO SystemMonitor:4098812 [interfaces.py:start():190] Started gpu monitoring +2024-03-13 11:28:08,688 INFO SystemMonitor:4098812 [interfaces.py:start():190] Started memory monitoring +2024-03-13 11:28:08,688 INFO SystemMonitor:4098812 [interfaces.py:start():190] Started network monitoring +2024-03-13 11:28:08,758 DEBUG HandlerThread:4098812 [system_info.py:probe():151] Probing system +2024-03-13 11:28:08,763 DEBUG HandlerThread:4098812 [system_info.py:_probe_git():136] Probing git +2024-03-13 11:28:08,783 DEBUG HandlerThread:4098812 [system_info.py:_probe_git():144] Probing git done +2024-03-13 11:28:08,784 DEBUG HandlerThread:4098812 [system_info.py:probe():199] Probing system done +2024-03-13 11:28:08,784 DEBUG HandlerThread:4098812 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-13T11:28:08.759036', 'startedAt': '2024-03-13T11:28:08.012584', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--stage', 'finetune', '--pretrained_checkpoint', '/home/lilei/prismatic-vlms/runs/llava-qformer/stage1_qformer2_256/checkpoints/latest-checkpoint.pt', '--model.model_id', 'one-stage+7b_qformer2_256', '--run_id', 's1_s2_qformer2_256', '--model.arch_specifier', 'qformer2_256'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'e87c4d04915bf8fad0d90f2ae79454df778ee2d3'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-91', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2574.1339137254904, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.13, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3247.899, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3241.552, 'min': 1500.0, 'max': 2550.0}, {'current': 3247.589, 'min': 1500.0, 'max': 2550.0}, {'current': 2894.364, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.686, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.453, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.784, 'min': 1500.0, 'max': 2550.0}, {'current': 3247.472, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 64.8645248413086}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942749023438}} +2024-03-13 11:28:08,784 INFO HandlerThread:4098812 [system_monitor.py:probe():224] Finished collecting system info +2024-03-13 11:28:08,784 INFO HandlerThread:4098812 [system_monitor.py:probe():227] Publishing system info +2024-03-13 11:28:08,784 DEBUG HandlerThread:4098812 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-13 11:28:09,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/conda-environment.yaml +2024-03-13 11:28:13,586 DEBUG HandlerThread:4098812 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-13 11:28:13,595 INFO HandlerThread:4098812 [system_monitor.py:probe():229] Finished publishing system info +2024-03-13 11:28:13,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: files +2024-03-13 11:28:13,600 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-13 11:28:13,601 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:13,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: python_packages +2024-03-13 11:28:13,612 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: python_packages +2024-03-13 11:28:13,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:13,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:13,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:13,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/conda-environment.yaml +2024-03-13 11:28:13,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-metadata.json +2024-03-13 11:28:13,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/requirements.txt +2024-03-13 11:28:13,894 DEBUG SenderThread:4098812 [sender.py:send():382] send: telemetry +2024-03-13 11:28:14,451 INFO wandb-upload_0:4098812 [upload_job.py:push():131] Uploaded file /tmp/tmp8hxc00cwwandb/yg8l2yot-wandb-metadata.json +2024-03-13 11:28:14,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:16,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:19,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:19,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:24,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:28,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:28,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:28,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:29,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:34,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:39,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:40,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:41,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/config.yaml +2024-03-13 11:28:41,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:43,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:43,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:43,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:28:46,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:47,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:47,447 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:28:47,447 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:47,451 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:47,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_created():271] file/dir created: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:28:49,678 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:51,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:51,668 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:28:51,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:51,673 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:51,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:28:52,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:53,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:56,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:28:56,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:28:56,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:28:56,057 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:28:56,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:28:57,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:28:58,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:28:58,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:28:58,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:28:58,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:01,494 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:01,494 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:01,495 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:01,500 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:01,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:03,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:03,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:06,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:06,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:06,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:06,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:06,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:07,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:08,689 DEBUG SystemMonitor:4098812 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-13 11:29:08,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:29:08,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:11,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:11,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:11,289 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:11,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:11,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:13,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:13,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:13,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:13,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:13,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:17,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:17,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:17,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:17,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:17,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:19,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:19,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:22,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:22,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:22,468 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:22,473 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:22,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:23,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:24,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:28,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:28,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:28,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:29,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:29,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:29,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:29,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:29,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:30,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:31,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:34,565 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:34,566 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:34,566 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:34,571 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:34,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:35,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:35,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:38,694 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:29:39,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:39,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:39,402 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:39,407 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:39,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:41,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:41,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:43,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:43,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:43,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:29:44,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:44,157 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:44,157 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:44,162 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:44,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:45,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:47,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:48,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:48,534 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:48,535 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:48,540 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:48,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:49,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:52,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:54,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:29:54,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:29:54,380 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:29:54,385 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:29:54,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:29:55,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:29:58,385 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:29:58,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:29:58,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:29:58,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:00,357 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:00,358 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:00,358 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:00,363 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:00,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:01,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:03,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:05,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:05,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:05,328 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:05,334 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:05,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:07,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:08,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:30:08,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:10,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:10,700 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:10,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:10,705 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:10,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:13,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:13,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:13,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:13,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:13,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:15,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:15,146 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:15,146 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:15,152 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:15,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:17,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:19,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:19,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:19,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:19,370 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:19,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:19,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:21,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:24,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:25,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:25,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:25,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:25,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:25,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:27,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:28,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:28,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:28,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:29,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:30,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:30,277 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:30,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:30,282 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:30,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:31,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:35,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:38,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:38,504 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:38,504 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:38,510 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:38,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:30:38,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:39,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:40,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:42,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:42,774 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:42,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:42,780 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:42,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:43,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:43,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:43,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:45,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:46,033 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:47,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:47,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:47,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:47,698 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:47,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:49,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:30:51,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:56,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:30:56,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:30:56,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:30:56,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:30:56,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:30:57,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:30:58,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:30:58,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:30:58,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:30:59,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:00,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:00,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:00,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:00,924 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:01,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:01,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:03,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:05,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:05,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:05,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:05,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:05,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:07,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:07,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:08,708 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:31:10,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:10,950 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:10,950 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:10,955 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:11,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:12,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:13,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:13,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:13,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:13,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:16,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:16,808 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:16,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:16,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:16,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:18,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:19,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:21,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:21,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:21,864 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:21,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:22,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:23,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:23,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:28,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:28,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:28,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:28,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:28,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:28,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:28,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:28,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:29,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:31,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:33,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:34,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:34,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:34,015 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:34,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:34,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:35,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:38,702 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:31:38,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:38,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:38,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:38,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:38,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:39,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:39,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:43,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:43,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:43,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:43,761 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:43,843 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:43,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:43,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:43,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:44,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:45,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:49,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:50,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:50,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:50,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:50,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:50,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:51,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:31:55,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:31:57,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:31:57,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:31:57,883 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:31:57,888 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:31:58,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:31:58,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:31:58,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:31:58,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:31:59,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:00,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:02,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:02,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:02,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:02,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:02,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:03,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:06,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:06,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:06,449 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:06,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:06,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:06,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:07,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:08,710 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:32:11,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:12,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:12,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:12,598 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:12,603 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:12,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:13,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:13,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:13,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:16,390 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:16,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:16,396 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:16,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:17,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:17,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:20,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:20,672 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:20,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:20,677 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:20,894 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:21,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:22,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:27,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:27,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:27,098 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:27,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:27,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:28,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:28,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:28,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:28,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:29,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:32,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:32,554 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:32,554 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:32,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:32,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:33,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:33,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:38,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:38,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:32:40,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:40,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:40,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:40,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:41,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:43,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:43,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:43,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:43,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:43,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:48,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:50,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:50,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:50,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:50,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:50,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:51,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:54,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:32:54,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:32:54,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:32:54,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:32:54,923 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:32:54,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:32:57,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:32:58,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:32:58,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:32:58,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:32:59,826 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:01,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:01,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:01,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:01,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:01,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:03,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:05,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:06,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:06,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:06,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:06,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:06,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:07,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:08,708 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:33:10,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:11,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:11,699 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:11,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:11,704 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:11,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:13,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:13,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:13,943 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:15,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:16,297 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:16,298 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:16,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:16,303 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:16,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:17,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:21,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:21,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:21,761 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:21,762 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:21,767 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:21,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:23,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:26,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:28,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:28,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:28,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:28,324 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:28,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:28,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:28,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:28,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:29,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:31,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:33,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:33,914 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:33,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:33,920 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:33,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:35,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:36,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:38,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:33:39,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:39,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:39,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:39,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:39,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:41,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:42,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:43,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:43,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:43,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:46,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:46,381 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:46,382 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:46,387 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:46,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:47,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:48,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:51,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:51,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:51,018 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:51,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:51,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:53,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:33:54,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:58,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:33:58,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:33:58,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:33:58,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:33:58,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:33:58,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:33:58,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:33:58,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:33:59,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:33:59,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:04,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:04,789 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:04,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:04,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:04,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:05,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:05,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:08,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:34:09,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:09,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:09,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:09,251 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:09,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:11,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:12,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:13,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:13,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:14,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:14,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:14,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:14,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:15,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:16,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:16,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:18,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:18,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:18,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:18,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:19,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:20,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:21,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:23,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:23,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:23,231 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:23,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:24,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:26,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:27,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:28,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:28,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:28,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:31,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:31,261 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:31,261 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:31,266 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:32,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:32,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:34,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:36,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:36,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:36,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:36,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:37,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:37,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:38,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:38,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:34:41,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:41,665 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:41,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:41,672 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:42,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:43,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:43,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:43,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:43,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:44,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:45,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:45,893 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:45,893 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:45,898 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:46,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:48,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:48,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:51,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:51,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:51,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:51,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:52,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:54,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:54,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:34:55,773 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:34:55,774 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:34:55,774 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:34:55,779 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:34:56,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:34:58,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:34:58,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:34:58,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:34:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:34:59,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:00,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:00,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:00,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:00,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:01,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:02,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:04,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:04,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:04,780 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:04,786 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:05,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:05,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:06,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:08,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:35:08,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:08,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:08,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:08,908 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:09,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:10,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:10,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:13,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:13,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:13,345 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:13,351 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:13,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:13,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:13,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:14,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:16,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:16,843 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:17,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:17,860 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:17,860 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:17,866 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:18,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:20,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:21,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:23,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:23,136 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:23,136 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:23,141 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:24,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:26,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:27,142 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:28,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:28,412 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:28,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:28,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:28,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:28,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:28,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:29,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:30,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:32,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:32,473 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:32,474 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:32,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:32,479 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:33,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:34,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:36,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:36,642 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:36,643 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:36,648 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:37,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:37,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:38,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:38,718 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:35:41,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:41,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:41,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:41,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:42,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:43,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:43,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:43,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:43,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:44,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:47,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:47,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:47,222 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:47,227 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:48,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:48,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:50,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:52,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:52,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:52,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:52,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:53,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:53,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:35:54,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:35:57,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:35:57,105 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:35:57,105 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:35:57,111 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:35:57,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:35:58,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:35:58,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:35:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:35:58,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:00,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:03,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:03,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:03,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:03,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:04,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:04,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:06,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:08,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:36:09,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:09,247 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:09,247 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:09,252 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:10,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:10,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:12,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:13,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:13,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:13,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:13,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:13,845 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:13,846 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:13,851 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:14,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:15,851 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:16,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:20,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:22,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:22,978 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:22,978 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:22,983 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:23,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:24,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:25,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:27,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:27,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:27,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:27,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:28,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:28,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:28,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:28,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:30,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:31,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:33,762 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:33,763 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:33,764 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:33,769 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:34,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:36,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:37,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:38,255 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:38,256 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:38,256 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:38,261 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:38,723 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:36:39,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:40,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:43,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:43,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:43,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:36:43,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:44,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:44,625 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:44,626 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:44,631 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:45,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:46,193 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:49,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:50,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:50,533 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:50,533 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:50,538 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:51,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:52,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:55,174 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:36:55,176 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:36:55,176 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:36:55,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:36:55,181 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:36:55,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:36:58,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:36:58,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:36:58,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:36:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:00,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:00,193 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:00,194 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:00,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:00,199 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:00,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:02,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:04,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:04,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:04,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:04,258 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:05,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:05,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:06,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:08,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:37:08,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:08,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:08,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:08,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:09,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:10,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:10,801 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:13,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:13,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:13,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:14,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:14,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:14,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:14,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:15,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:16,225 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:16,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:19,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:19,914 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:19,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:19,920 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:20,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:21,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:22,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:24,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:24,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:24,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:24,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:25,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:26,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:27,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:28,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:28,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:28,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:32,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:32,598 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:32,599 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:32,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:32,604 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:33,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:34,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:37,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:37,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:37,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:37,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:38,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:38,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:38,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:37:40,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:43,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:43,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:43,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:43,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:43,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:43,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:43,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:44,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:46,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:47,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:47,309 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:47,309 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:47,314 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:48,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:49,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:50,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:54,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:55,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:55,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:55,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:55,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:37:56,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:37:58,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:37:58,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:37:58,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:37:58,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:37:59,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:37:59,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:37:59,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:37:59,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:37:59,952 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:00,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:02,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:03,822 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:03,823 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:03,823 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:03,828 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:04,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:05,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:06,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:08,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:08,182 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:08,182 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:08,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:08,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:08,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:38:10,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:11,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:12,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:12,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:12,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:12,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:13,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:13,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:13,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:14,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:16,847 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:18,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:18,162 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:18,162 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:18,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:18,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:20,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:22,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:22,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:22,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:22,542 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:22,547 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:23,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:24,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:26,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:26,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:26,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:26,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:27,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:27,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:28,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:28,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:28,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:28,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:32,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:32,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:32,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:32,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:32,970 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:33,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:34,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:38,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:38,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:38,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:38,030 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:38,035 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:38,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:38,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:38:40,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:43,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:43,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:43,917 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:43,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:43,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:43,929 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:43,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:44,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:46,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:48,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:52,527 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:52,528 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:52,528 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:52,533 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:53,336 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:54,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:54,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:38:56,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:38:56,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:38:56,975 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:38:56,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:38:57,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:38:58,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:38:58,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:38:58,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:38:58,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:38:59,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:03,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:03,136 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:03,136 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:03,141 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:03,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:04,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:05,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:08,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:39:08,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:08,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:08,872 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:08,877 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:09,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:10,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:10,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:13,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:13,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:13,273 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:13,278 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:13,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:13,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:13,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:14,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:15,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:19,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:19,288 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:19,288 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:19,293 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:19,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:21,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:22,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:24,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:24,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:24,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:24,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:25,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:26,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:26,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:28,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:28,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:28,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:31,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:31,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:31,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:31,070 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:31,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:32,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:32,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:35,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:35,442 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:35,443 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:35,447 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:36,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:37,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:38,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:38,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:39:39,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:39,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:39,868 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:39,873 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:40,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:42,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:42,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:43,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:43,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:45,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:45,339 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:45,339 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:45,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:45,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:48,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:48,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:50,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:50,074 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:50,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:50,079 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:50,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:52,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:54,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:39:57,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:39:57,057 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:39:57,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:39:57,063 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:39:57,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:39:58,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:39:58,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:39:58,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:39:58,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:39:59,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:01,367 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:01,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:01,368 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:01,373 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:01,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:04,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:05,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:06,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:06,032 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:06,032 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:06,038 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:06,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:08,436 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:08,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:40:10,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:10,509 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:10,509 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:10,509 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:10,514 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:11,438 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:12,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:13,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:13,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:15,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:16,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:16,042 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:16,043 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:16,048 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:16,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:18,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:20,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:20,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:20,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:20,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:20,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:21,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:22,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:24,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:24,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:24,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:24,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:25,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:26,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:26,658 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:28,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:28,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:28,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:31,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:31,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:31,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:31,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:31,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:32,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:32,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:35,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:35,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:35,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:35,659 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:36,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:37,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:38,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:38,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:40:40,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:40,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:40,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:40,739 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:41,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:42,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:42,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:43,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:43,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:44,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:44,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:44,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:44,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:45,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:46,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:48,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:50,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:50,232 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:50,232 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:50,237 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:50,480 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:52,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:53,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:54,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:54,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:54,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:54,067 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:54,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:40:56,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:40:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:40:58,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:40:58,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:40:58,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:40:58,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:40:58,880 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:40:58,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:40:58,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:40:59,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:00,494 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:02,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:02,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:02,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:02,923 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:03,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:03,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:04,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:06,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:06,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:06,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:06,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:07,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:08,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:08,741 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:41:09,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:13,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:13,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:13,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:14,865 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:16,385 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:16,386 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:16,386 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:16,391 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:16,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:18,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:20,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:24,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:24,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:24,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:24,096 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:24,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:26,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:26,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:28,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:28,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:28,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:31,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:32,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:32,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:32,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:32,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:33,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:34,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:37,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:38,743 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:41:40,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:40,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:40,878 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:40,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:41,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:42,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:42,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:43,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:43,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:43,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:44,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:44,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:44,998 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:45,003 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:45,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:46,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:48,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:50,243 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:50,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:50,244 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:50,249 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:50,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:52,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:53,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:54,761 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:54,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:54,762 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:54,767 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:55,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:41:56,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:41:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:41:58,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:41:58,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:41:58,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:41:59,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:41:59,424 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:41:59,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:41:59,429 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:41:59,551 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:00,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:04,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:04,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:04,817 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:04,817 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:04,822 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:05,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:06,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:08,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:42:09,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:09,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:09,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:09,864 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:09,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:10,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:12,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:13,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:13,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:13,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:14,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:14,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:14,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:14,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:14,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:15,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:16,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:19,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:20,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:20,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:20,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:20,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:21,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:22,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:24,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:25,481 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:25,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:25,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:25,488 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:25,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:26,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:28,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:28,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:28,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:29,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:30,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:30,528 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:30,529 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:30,534 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:30,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:32,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:35,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:38,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:42:38,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:38,992 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:38,993 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:38,998 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:39,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:40,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:40,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:43,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:43,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:43,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:43,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:43,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:43,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:43,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:44,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:46,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:48,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:48,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:48,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:48,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:48,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:50,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:52,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:52,293 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:52,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:52,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:52,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:52,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:42:54,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:42:57,299 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:42:57,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:42:57,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:42:57,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:42:57,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:42:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:42:58,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:42:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:42:58,656 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:00,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:02,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:02,604 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:02,604 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:02,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:02,609 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:02,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:04,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:07,083 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:07,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:07,084 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:07,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:07,668 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:08,090 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:08,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:08,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:43:12,175 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:12,176 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:12,176 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:12,181 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:12,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:13,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:13,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:13,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:13,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:14,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:18,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:18,212 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:18,213 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:18,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:18,218 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:18,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:20,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:23,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:23,072 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:23,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:23,078 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:23,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:24,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:24,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:27,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:27,604 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:27,604 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:27,609 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:27,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:28,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:28,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:28,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:29,839 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:30,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:32,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:32,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:32,121 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:32,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:32,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:34,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:35,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:36,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:36,808 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:36,809 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:36,814 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:37,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:38,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:38,752 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:43:40,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:41,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:41,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:41,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:41,260 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:41,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:42,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:43,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:43:45,866 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:49,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:49,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:49,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:49,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:49,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:50,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:51,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:54,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:43:54,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:43:54,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:43:54,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:43:55,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:43:56,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:43:56,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:43:58,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:43:58,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:43:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:00,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:00,603 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:00,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:00,609 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:00,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:02,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:02,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:04,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:04,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:04,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:04,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:05,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:06,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:07,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:08,760 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:44:10,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:10,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:10,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:10,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:11,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:12,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:13,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:13,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:13,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:13,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:15,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:15,869 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:15,869 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:15,874 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:16,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:18,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:18,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:23,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:24,226 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:24,226 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:24,227 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:24,232 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:24,768 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:26,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:28,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:28,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:28,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:29,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:29,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:29,321 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:29,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:29,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:29,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:30,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:34,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:34,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:34,471 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:34,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:34,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:34,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:36,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:38,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:44:39,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:39,592 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:39,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:39,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:39,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:39,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:40,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:43,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:44,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:44,620 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:44,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:44,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:44,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:44,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:46,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:49,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:50,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:50,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:50,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:50,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:50,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:52,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:54,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:54,811 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:54,811 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:54,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:44:54,816 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:55,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:56,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:44:58,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:44:58,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:44:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:44:58,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:44:58,845 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:44:58,845 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:44:58,850 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:44:59,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:44:59,851 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:00,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:03,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:03,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:03,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:03,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:03,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:04,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:05,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:08,758 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:45:09,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:09,184 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:09,185 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:09,190 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:09,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:10,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:11,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:13,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:13,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:13,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:15,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:15,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:15,353 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:15,358 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:15,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:16,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:16,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:19,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:19,870 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:19,871 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:19,875 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:20,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:21,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:22,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:24,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:24,133 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:24,133 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:24,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:24,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:26,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:27,139 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:28,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:28,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:28,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:30,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:30,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:30,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:30,472 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:30,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:32,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:32,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:35,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:35,567 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:35,567 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:35,572 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:35,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:36,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:37,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:38,760 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:45:40,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:40,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:40,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:40,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:40,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:42,802 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:42,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:43,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:43,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:46,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:46,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:46,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:46,269 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:46,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:48,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:48,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:50,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:50,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:50,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:50,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:51,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:52,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:53,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:45:55,729 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:45:55,730 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:45:55,730 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:45:55,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:45:55,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:45:56,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:45:58,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:45:58,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:45:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:45:59,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:00,822 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:00,822 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:00,823 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:00,828 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:00,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:02,876 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:04,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:05,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:05,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:05,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:05,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:06,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:08,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:46:08,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:10,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:11,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:11,033 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:11,033 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:11,039 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:11,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:12,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:13,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:13,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:13,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:15,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:15,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:15,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:15,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:15,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:16,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:16,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:20,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:20,580 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:20,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:20,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:20,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:21,585 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:22,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:25,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:25,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:25,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:25,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:25,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:26,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:27,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:28,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:28,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:30,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:30,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:30,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:30,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:30,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:32,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:32,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:35,133 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:35,134 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:35,134 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:35,140 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:35,913 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:36,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:38,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:38,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:46:39,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:39,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:39,833 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:39,838 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:39,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:42,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:43,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:43,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:43,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:43,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:44,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:44,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:44,278 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:44,283 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:44,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:46,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:49,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:53,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:53,447 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:53,447 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:53,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:53,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:54,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:46:54,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:46:58,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:46:58,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:46:58,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:46:58,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:46:58,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:46:58,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:46:58,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:46:58,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:46:59,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:00,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:04,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:06,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:06,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:06,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:06,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:06,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:08,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:47:09,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:10,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:10,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:10,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:10,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:10,445 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:11,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:13,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:13,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:13,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:13,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:15,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:15,221 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:15,221 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:15,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:16,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:16,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:17,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:21,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:23,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:23,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:23,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:23,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:24,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:25,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:26,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:28,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:28,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:28,802 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:28,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:28,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:28,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:29,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:31,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:31,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:34,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:34,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:34,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:34,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:35,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:37,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:37,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:38,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:47:39,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:39,261 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:39,261 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:39,266 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:40,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:41,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:43,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:43,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:43,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:43,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:45,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:45,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:45,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:45,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:46,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:49,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:49,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:50,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:50,464 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:50,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:50,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:51,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:53,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:54,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:47:56,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:47:56,590 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:47:56,591 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:47:56,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:47:57,081 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:47:58,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:47:58,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:47:58,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:47:59,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:47:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:00,837 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:00,838 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:00,838 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:00,843 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:01,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:03,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:05,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:05,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:05,892 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:05,892 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:05,897 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:06,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:07,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:08,771 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:48:10,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:10,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:10,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:10,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:11,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:11,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:13,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:13,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:13,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:13,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:14,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:14,639 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:14,639 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:14,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:15,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:16,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:17,104 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:19,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:19,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:19,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:19,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:20,106 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:21,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:21,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:24,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:24,885 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:24,885 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:24,890 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:25,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:27,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:27,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:28,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:28,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:29,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:29,545 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:29,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:29,550 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:30,114 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:31,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:33,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:38,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:38,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:38,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:38,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:38,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:38,773 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:48:38,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:41,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:43,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:43,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:43,251 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:43,256 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:43,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:43,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:43,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:44,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:45,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:47,318 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:47,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:47,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:47,324 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:48,134 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:49,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:49,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:51,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:51,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:51,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:51,649 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:52,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:53,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:54,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:48:55,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:48:55,879 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:48:55,879 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:48:55,884 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:48:56,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:48:57,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:48:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:48:58,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:48:58,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:48:59,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:00,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:00,876 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:00,876 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:00,881 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:01,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:03,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:04,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:05,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:05,133 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:05,133 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:05,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:05,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:07,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:08,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:49:09,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:09,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:09,757 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:09,762 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:10,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:10,762 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:11,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:13,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:13,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:13,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:14,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:14,435 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:14,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:14,440 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:15,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:16,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:17,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:21,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:22,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:22,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:22,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:22,758 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:23,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:25,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:26,759 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:28,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:28,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:28,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:30,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:30,103 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:30,103 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:30,108 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:30,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:32,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:33,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:37,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:38,777 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:49:39,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:39,699 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:39,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:39,705 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:40,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:41,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:42,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:43,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:43,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:44,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:44,389 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:44,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:44,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:45,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:47,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:48,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:53,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:53,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:53,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:53,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:53,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:54,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:49:55,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:57,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:49:57,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:49:57,748 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:49:57,753 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:49:58,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:49:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:49:58,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:49:58,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:49:59,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:49:59,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:01,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:01,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:01,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:01,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:02,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:03,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:04,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:06,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:06,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:06,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:06,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:07,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:08,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:50:09,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:10,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:11,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:11,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:11,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:11,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:11,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:13,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:13,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:13,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:13,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:15,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:15,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:15,821 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:15,821 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:15,826 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:16,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:17,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:20,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:21,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:21,579 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:21,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:21,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:22,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:23,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:25,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:25,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:25,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:25,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:25,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:26,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:27,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:28,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:28,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:29,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:29,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:29,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:29,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:30,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:30,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:31,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:35,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:36,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:36,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:36,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:36,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:37,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:38,781 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:50:39,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:41,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:42,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:43,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:43,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:43,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:43,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:43,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:43,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:45,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:46,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:48,563 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:48,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:48,564 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:48,569 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:49,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:51,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:52,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:52,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:52,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:52,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:52,690 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:53,280 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:55,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:50:57,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:50:57,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:50:57,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:50:57,125 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:50:57,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:50:58,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:50:58,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:50:58,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:50:58,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:50:59,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:03,375 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:03,375 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:03,376 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:03,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:03,380 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:04,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:05,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:08,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:08,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:08,015 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:08,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:08,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:08,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:51:08,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:09,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:12,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:12,124 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:12,124 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:12,129 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:12,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:13,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:13,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:13,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:13,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:13,886 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:16,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:16,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:16,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:17,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:19,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:19,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:24,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:25,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:25,620 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:25,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:25,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:26,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:27,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:28,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:28,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:29,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:30,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:30,310 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:30,311 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:30,316 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:30,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:33,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:34,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:34,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:34,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:34,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:35,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:35,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:37,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:38,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:51:40,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:40,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:40,258 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:40,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:40,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:41,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:43,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:43,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:43,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:43,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:51:44,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:44,711 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:44,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:44,717 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:45,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:46,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:47,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:51,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:51,631 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:51,632 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:51,637 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:52,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:52,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:53,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:56,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:51:56,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:51:56,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:51:56,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:51:56,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:51:57,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:51:58,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:51:58,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:51:58,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:51:58,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:01,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:01,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:01,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:01,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:02,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:03,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:03,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:06,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:06,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:06,782 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:06,787 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:07,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:08,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:52:09,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:09,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:12,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:12,882 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:12,882 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:12,888 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:13,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:13,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:13,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:13,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:14,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:15,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:17,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:17,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:17,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:17,711 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:18,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:19,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:20,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:22,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:22,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:22,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:22,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:23,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:25,372 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:25,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:27,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:27,454 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:27,454 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:27,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:28,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:28,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:28,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:29,377 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:31,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:31,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:31,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:31,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:31,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:32,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:33,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:36,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:36,664 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:36,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:36,669 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:37,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:37,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:38,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:52:39,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:41,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:41,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:41,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:41,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:42,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:42,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:43,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:43,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:43,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:43,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:47,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:48,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:48,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:48,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:48,207 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:48,396 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:49,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:53,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:53,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:53,117 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:53,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:53,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:53,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:55,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:52:57,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:52:57,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:52:57,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:52:57,977 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:52:58,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:52:58,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:52:58,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:52:58,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:52:58,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:52:59,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:03,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:03,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:03,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:03,059 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:03,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:04,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:05,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:07,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:07,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:07,872 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:07,877 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:08,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:08,792 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:53:09,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:09,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:13,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:13,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:13,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:14,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:16,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:16,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:16,584 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:16,589 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:17,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:19,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:20,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:21,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:21,130 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:21,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:21,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:21,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:23,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:25,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:25,572 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:25,573 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:25,578 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:26,436 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:26,578 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:27,438 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:28,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:28,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:31,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:31,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:31,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:31,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:31,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:32,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:33,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:36,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:36,232 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:36,233 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:36,238 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:36,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:37,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:38,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:38,797 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:53:40,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:40,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:40,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:40,428 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:40,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:43,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:43,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:43,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:43,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:43,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:44,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:44,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:44,940 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:44,945 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:45,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:47,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:48,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:49,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:49,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:49,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:49,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:50,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:51,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:53,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:53:53,982 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:53:53,983 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:53:53,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:53:53,989 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:53:54,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:53:55,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:53:58,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:53:58,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:53:58,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:53:59,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:00,730 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:00,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:00,731 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:00,736 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:01,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:03,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:05,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:07,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:07,305 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:07,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:07,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:07,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:08,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:54:09,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:10,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:12,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:12,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:12,404 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:12,409 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:12,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:13,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:13,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:13,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:15,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:15,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:18,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:18,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:18,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:18,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:19,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:21,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:21,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:24,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:24,108 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:24,108 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:24,114 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:24,497 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:25,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:27,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:28,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:28,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:28,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:30,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:30,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:30,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:30,063 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:30,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:31,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:33,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:34,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:34,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:34,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:34,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:34,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:37,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:38,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:38,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:54:38,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:38,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:38,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:38,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:39,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:41,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:43,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:43,326 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:43,326 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:43,331 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:43,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:43,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:43,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:43,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:43,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:45,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:48,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:48,639 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:48,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:48,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:49,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:49,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:51,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:54,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:54,001 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:54,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:54,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:54,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:54:55,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:54:55,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:54:58,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:54:58,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:54:58,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:54:59,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:54:59,288 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:54:59,288 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:54:59,294 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:54:59,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:00,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:01,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:04,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:04,075 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:04,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:04,080 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:04,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:05,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:06,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:08,800 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:55:11,802 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:12,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:12,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:12,186 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:12,191 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:12,557 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:13,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:13,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:13,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:13,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:16,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:21,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:21,491 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:21,491 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:21,496 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:21,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:22,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:23,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:27,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:28,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:28,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:28,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:29,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:29,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:29,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:29,804 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:30,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:31,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:32,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:37,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:38,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:38,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:38,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:38,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:38,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:38,802 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:55:39,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:43,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:43,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:43,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:43,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:44,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:44,384 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:44,384 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:44,389 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:44,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:45,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:49,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:52,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:52,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:52,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:52,695 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:53,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:54,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:55:55,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:57,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:55:57,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:55:57,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:55:57,304 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:55:57,601 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:55:58,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:55:58,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:55:58,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:55:59,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:55:59,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:01,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:01,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:01,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:01,421 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:01,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:03,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:05,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:07,527 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:07,528 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:07,528 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:07,538 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:07,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:08,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:56:09,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:10,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:12,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:12,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:12,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:12,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:13,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:13,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:13,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:13,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:15,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:15,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:18,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:18,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:18,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:18,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:18,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:19,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:21,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:24,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:24,012 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:24,012 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:24,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:24,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:25,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:27,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:28,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:28,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:28,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:28,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:28,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:28,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:28,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:29,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:31,660 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:32,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:33,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:33,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:33,469 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:33,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:33,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:35,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:38,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:38,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:56:38,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:38,815 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:38,816 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:38,822 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:39,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:41,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:43,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:43,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:43,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:43,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:44,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:44,989 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:44,989 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:44,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:45,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:47,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:49,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:49,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:49,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:49,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:49,716 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:50,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:51,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:54,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:56:54,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:56:54,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:56:54,992 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:56:55,001 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:56:55,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:56:57,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:56:58,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:56:58,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:56:58,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:56:59,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:03,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:03,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:03,329 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:03,336 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:03,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:05,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:05,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:08,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:08,445 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:08,445 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:08,451 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:08,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:08,809 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:57:09,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:10,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:12,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:12,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:12,637 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:12,643 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:12,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:13,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:13,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:13,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:15,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:16,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:20,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:20,119 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:20,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:20,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:20,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:21,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:22,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:25,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:25,144 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:25,145 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:25,153 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:25,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:27,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:28,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:28,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:28,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:28,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:33,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:33,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:33,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:33,565 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:33,571 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:33,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:35,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:38,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:38,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:38,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:38,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:38,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:38,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:38,811 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:57:39,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:42,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:42,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:42,874 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:42,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:43,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:43,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:43,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:43,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:44,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:45,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:47,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:47,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:47,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:47,977 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:48,790 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:49,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:49,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:52,586 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:52,587 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:52,587 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:52,593 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:52,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:53,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:57:54,594 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:57:57,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:57:57,060 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:57:57,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:57:57,069 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:57:57,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:57:58,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:57:58,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:57:58,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:57:59,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:00,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:01,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:01,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:01,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:01,439 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:01,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:03,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:05,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:07,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:07,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:07,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:07,085 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:07,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:08,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:58:09,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:10,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:13,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:13,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:13,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:14,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:14,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:14,194 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:14,203 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:14,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:14,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:16,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:19,554 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:19,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:19,555 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:19,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:19,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:20,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:21,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:24,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:24,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:24,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:24,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:24,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:26,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:26,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:28,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:28,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:28,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:31,225 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:31,226 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:31,226 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:31,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:31,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:32,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:32,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:36,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:36,419 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:36,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:36,425 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:36,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:36,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:37,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:38,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:58:40,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:40,962 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:40,963 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:40,968 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:41,876 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:42,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:42,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:43,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:43,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:43,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:58:46,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:46,752 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:46,752 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:46,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:46,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:48,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:48,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:51,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:51,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:51,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:51,447 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:51,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:52,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:54,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:58:55,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:58:55,549 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:58:55,549 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:58:55,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:58:55,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:58:56,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:58:58,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:58:58,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:58:58,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:00,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:01,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:01,017 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:01,018 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:01,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:01,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:02,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:06,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:06,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:06,794 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:06,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:06,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:06,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:08,817 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:59:08,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:11,819 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:12,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:12,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:12,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:12,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:12,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:12,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:13,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:13,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:14,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:17,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:17,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:17,159 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:17,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:17,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:17,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:18,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:21,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:21,572 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:21,573 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:21,578 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:21,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:22,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:22,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:25,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:25,814 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:25,814 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:25,820 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:25,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:26,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:27,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:28,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:28,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:28,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:30,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:30,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:30,108 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:30,114 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:30,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:30,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:33,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:34,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:34,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:34,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:34,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:34,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:34,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:38,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:38,826 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 11:59:40,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:40,592 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:40,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:40,598 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:40,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:40,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:43,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:43,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:43,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 11:59:43,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:45,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:45,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:45,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:45,639 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:45,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:46,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:49,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:52,379 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:52,380 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:52,380 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:52,386 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:52,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 11:59:52,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:55,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 11:59:58,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 11:59:58,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 11:59:58,750 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 11:59:58,756 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 11:59:58,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 11:59:58,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 11:59:58,983 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 11:59:59,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:00,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:01,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:05,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:05,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:05,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:05,564 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:05,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:06,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:06,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:08,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:00:11,823 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:11,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:11,914 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:11,914 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:11,919 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:11,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:13,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:13,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:13,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:13,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:17,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:17,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:17,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:17,650 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:17,655 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:18,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:19,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:22,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:22,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:22,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:22,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:22,718 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:23,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:23,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:27,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:27,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:27,812 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:27,812 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:27,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:28,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:28,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:28,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:28,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:29,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:33,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:33,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:33,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:33,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:33,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:34,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:35,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:38,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:38,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:38,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:38,548 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:38,554 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:38,823 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:00:39,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:39,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:43,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:43,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:43,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:43,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:45,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:45,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:45,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:45,285 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:46,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:47,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:49,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:53,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:53,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:53,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:53,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:54,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:00:54,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:00:55,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:58,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:00:58,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:00:58,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:00:58,173 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:00:58,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:00:58,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:00:58,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:00:59,058 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:00:59,058 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:00,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:02,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:02,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:02,888 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:02,894 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:03,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:05,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:05,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:07,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:07,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:07,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:07,209 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:08,068 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:08,826 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:01:09,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:11,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:12,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:12,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:12,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:12,973 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:13,073 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:13,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:13,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:13,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:15,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:17,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:17,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:17,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:17,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:17,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:18,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:19,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:21,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:21,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:21,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:21,799 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:22,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:22,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:23,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:26,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:26,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:26,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:26,577 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:27,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:27,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:28,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:28,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:28,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:28,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:31,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:31,972 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:31,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:31,977 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:32,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:33,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:34,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:36,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:36,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:36,842 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:36,847 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:37,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:37,104 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:38,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:01:39,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:42,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:42,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:42,321 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:42,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:43,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:43,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:43,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:43,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:43,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:45,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:46,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:46,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:46,638 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:46,643 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:47,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:47,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:50,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:52,790 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:52,791 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:52,792 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:52,797 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:53,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:01:53,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:55,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:01:57,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:01:57,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:01:57,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:01:57,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:01:58,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:01:58,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:01:58,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:01:58,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:01:59,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:01,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:03,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:03,546 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:03,546 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:03,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:04,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:05,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:06,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:08,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:02:11,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:11,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:11,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:11,928 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:11,933 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:12,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:13,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:13,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:13,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:13,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:17,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:20,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:20,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:20,222 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:20,227 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:21,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:21,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:22,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:24,449 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:24,450 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:24,450 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:24,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:25,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:25,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:27,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:28,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:28,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:28,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:30,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:30,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:30,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:30,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:31,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:31,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:32,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:34,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:34,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:34,888 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:34,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:35,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:35,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:37,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:38,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:02:39,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:39,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:39,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:39,134 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:39,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:41,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:43,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:43,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:43,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:02:43,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:44,518 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:44,519 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:44,519 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:44,524 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:45,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:45,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:48,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:49,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:49,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:49,479 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:49,484 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:50,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:51,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:54,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:02:54,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:02:54,607 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:02:54,607 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:02:54,612 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:02:55,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:02:55,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:02:58,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:02:58,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:02:58,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:00,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:05,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:05,127 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:05,127 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:05,132 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:05,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:06,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:07,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:08,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:03:11,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:13,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:13,700 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:13,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:13,706 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:13,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:13,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:13,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:14,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:15,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:17,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:21,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:21,869 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:21,870 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:21,875 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:22,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:22,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:23,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:26,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:26,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:26,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:26,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:27,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:27,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:28,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:28,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:28,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:28,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:32,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:32,123 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:32,124 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:32,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:32,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:33,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:34,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:36,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:36,857 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:36,857 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:36,863 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:37,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:37,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:38,837 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:03:39,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:40,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:40,966 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:40,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:40,972 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:41,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:41,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:43,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:43,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:43,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:45,266 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:45,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:45,349 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:45,349 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:45,355 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:46,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:47,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:49,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:49,486 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:49,486 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:49,491 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:50,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:50,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:51,253 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:53,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:53,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:53,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:53,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:03:54,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:03:55,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:03:55,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:03:58,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:03:58,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:03:58,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:03:59,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:03:59,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:03:59,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:03:59,442 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:00,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:01,264 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:01,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:05,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:05,797 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:05,797 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:05,802 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:06,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:06,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:07,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:08,839 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:04:11,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:12,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:12,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:12,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:12,994 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:13,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:13,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:13,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:13,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:13,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:17,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:17,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:17,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:17,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:17,444 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:18,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:19,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:21,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:21,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:21,990 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:21,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:22,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:22,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:23,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:27,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:28,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:28,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:28,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:28,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:28,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:28,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:28,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:29,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:29,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:32,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:32,905 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:32,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:32,911 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:33,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:33,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:33,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:37,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:37,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:37,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:37,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:38,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:38,844 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:04:39,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:39,845 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:42,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:42,500 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:42,501 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:42,506 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:43,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:43,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:43,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:43,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:44,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:45,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:47,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:47,786 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:47,786 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:47,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:48,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:49,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:50,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:54,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:54,221 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:54,221 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:54,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:54,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:04:55,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:04:56,226 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:04:58,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:04:58,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:04:59,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:04:59,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:04:59,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:04:59,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:04:59,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:04:59,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:01,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:02,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:03,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:03,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:03,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:03,561 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:04,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:05,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:07,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:08,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:05:09,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:09,680 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:09,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:09,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:10,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:11,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:12,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:13,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:13,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:14,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:18,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:18,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:18,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:18,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:18,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:18,452 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:19,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:23,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:26,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:26,346 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:26,346 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:26,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:26,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:27,461 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:28,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:28,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:28,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:29,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:31,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:31,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:31,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:31,206 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:31,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:33,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:34,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:35,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:35,579 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:35,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:35,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:36,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:37,471 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:38,847 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:05:39,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:39,815 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:39,816 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:39,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:39,821 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:40,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:41,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:43,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:43,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:43,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:44,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:44,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:44,928 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:44,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:44,933 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:45,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:45,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:49,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:53,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:53,920 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:53,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:53,926 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:54,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:55,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:05:55,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:05:57,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:05:57,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:05:57,789 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:05:57,794 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:05:58,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:05:58,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:05:58,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:05:59,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:05:59,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:01,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:02,465 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:02,465 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:02,465 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:02,470 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:02,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:03,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:06,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:07,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:07,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:07,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:07,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:07,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:07,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:08,847 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:06:11,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:11,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:11,719 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:11,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:11,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:12,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:13,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:13,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:13,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:13,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:16,442 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:16,443 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:16,443 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:16,448 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:16,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:17,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:17,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:20,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:20,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:20,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:20,805 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:21,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:21,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:22,805 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:27,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:27,346 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:27,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:27,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:27,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:28,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:28,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:28,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:29,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:29,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:34,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:35,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:35,790 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:35,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:35,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:36,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:37,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:38,849 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:06:39,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:40,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:40,044 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:40,044 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:40,050 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:40,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:41,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:43,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:43,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:43,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:06:45,243 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:46,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:46,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:46,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:46,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:46,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:47,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:50,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:50,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:50,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:50,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:50,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:51,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:51,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:55,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:06:55,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:06:55,830 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:06:55,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:06:55,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:06:56,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:06:57,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:06:58,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:06:58,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:06:59,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:01,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:05,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:05,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:05,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:05,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:05,592 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:05,592 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:07,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:08,851 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:07:09,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:09,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:09,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:09,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:10,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:11,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:12,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:13,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:13,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:14,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:16,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:16,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:16,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:16,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:16,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:17,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:18,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:20,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:20,340 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:20,340 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:20,346 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:20,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:21,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:23,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:25,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:25,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:25,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:25,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:25,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:27,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:28,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:28,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:28,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:28,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:30,757 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:30,758 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:30,758 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:30,763 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:31,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:31,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:34,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:36,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:36,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:36,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:36,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:36,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:37,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:38,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:07:39,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:41,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:41,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:41,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:41,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:42,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:43,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:43,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:43,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:44,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:07:45,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:46,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:46,143 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:46,143 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:46,149 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:46,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:47,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:50,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:51,497 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:51,498 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:51,498 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:51,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:51,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:53,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:55,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:07:55,730 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:07:55,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:07:55,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:07:55,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:07:56,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:07:57,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:07:58,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:07:58,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:07:58,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:00,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:00,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:00,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:00,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:00,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:01,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:01,659 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:05,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:05,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:05,884 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:05,889 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:06,663 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:06,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:07,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:08,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:08:10,367 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:10,368 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:10,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:10,374 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:10,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:11,671 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:12,375 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:13,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:13,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:14,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:17,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:17,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:17,196 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:17,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:17,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:17,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:18,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:21,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:21,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:21,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:21,946 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:22,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:23,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:23,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:26,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:26,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:26,961 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:26,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:27,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:27,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:28,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:28,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:28,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:29,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:34,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:35,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:35,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:35,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:35,908 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:36,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:37,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:38,857 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:08:39,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:41,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:41,870 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:41,871 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:41,876 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:42,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:43,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:43,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:43,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:44,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:08:45,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:46,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:46,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:46,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:46,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:46,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:47,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:50,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:51,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:51,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:51,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:51,400 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:51,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:53,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:55,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:08:55,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:08:55,927 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:08:55,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:08:55,932 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:08:56,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:08:57,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:08:58,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:08:58,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:08:58,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:01,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:02,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:02,257 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:02,257 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:02,262 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:02,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:03,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:06,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:07,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:07,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:07,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:07,172 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:07,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:07,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:08,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:09:11,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:12,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:12,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:12,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:12,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:12,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:13,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:13,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:13,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:13,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:16,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:16,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:16,981 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:16,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:16,986 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:17,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:17,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:21,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:21,851 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:21,852 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:21,857 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:22,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:22,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:23,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:26,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:26,057 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:26,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:26,062 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:26,768 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:27,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:28,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:28,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:28,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:29,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:30,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:30,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:30,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:30,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:30,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:31,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:33,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:36,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:36,422 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:36,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:36,427 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:36,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:37,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:38,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:38,862 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:09:40,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:40,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:40,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:40,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:40,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:41,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:43,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:43,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:43,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:44,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:48,554 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:48,554 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:48,554 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:48,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:48,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:49,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:49,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:53,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:53,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:53,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:53,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:53,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:09:53,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:55,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:09:58,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:09:58,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:09:58,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:09:58,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:09:58,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:09:58,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:09:58,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:09:58,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:09:59,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:00,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:03,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:03,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:03,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:03,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:03,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:05,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:05,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:08,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:10:10,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:10,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:10,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:10,240 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:10,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:11,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:11,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:13,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:13,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:13,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:15,225 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:15,226 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:15,226 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:15,231 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:15,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:15,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:16,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:19,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:19,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:19,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:19,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:19,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:21,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:21,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:24,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:24,892 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:24,892 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:24,898 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:25,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:25,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:26,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:28,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:28,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:29,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:32,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:33,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:33,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:33,321 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:33,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:33,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:33,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:37,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:38,458 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:38,459 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:38,459 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:38,464 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:38,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:38,866 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:10:39,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:42,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:43,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:43,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:43,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:10:44,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:44,516 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:44,516 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:44,522 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:44,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:45,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:48,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:50,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:50,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:50,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:50,284 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:50,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:51,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:54,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:10:57,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:10:57,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:10:57,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:10:57,083 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:10:57,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:10:57,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:10:58,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:10:58,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:10:59,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:00,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:05,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:05,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:05,501 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:05,501 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:05,510 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:05,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:07,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:08,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:11:09,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:09,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:09,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:09,910 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:10,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:10,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:11,876 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:13,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:13,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:13,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:16,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:16,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:16,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:16,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:16,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:16,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:17,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:20,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:20,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:20,585 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:20,590 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:20,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:21,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:21,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:24,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:24,870 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:24,871 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:24,876 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:24,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:25,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:26,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:28,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:28,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:28,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:30,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:30,276 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:30,276 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:30,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:30,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:31,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:32,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:35,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:35,085 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:35,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:35,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:35,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:35,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:38,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:38,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:11:39,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:39,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:39,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:39,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:40,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:41,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:43,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:43,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:43,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:44,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:11:47,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:47,019 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:47,020 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:47,025 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:47,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:47,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:49,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:50,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:50,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:50,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:50,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:50,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:51,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:54,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:11:55,286 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:11:55,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:11:55,287 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:11:55,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:11:55,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:11:55,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:11:58,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:11:58,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:11:58,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:00,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:00,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:00,098 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:00,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:00,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:00,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:01,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:05,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:05,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:05,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:05,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:05,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:05,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:07,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:08,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:12:10,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:11,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:11,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:11,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:11,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:11,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:13,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:13,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:13,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:13,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:16,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:18,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:18,752 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:18,752 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:18,757 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:18,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:19,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:21,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:24,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:24,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:24,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:24,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:24,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:25,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:27,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:28,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:28,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:29,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:31,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:31,711 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:31,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:31,716 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:31,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:32,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:33,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:37,611 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:37,612 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:37,612 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:37,618 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:37,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:38,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:38,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:12:39,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:42,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:42,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:42,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:42,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:42,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:43,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:43,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:43,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:43,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:43,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:46,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:46,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:46,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:46,618 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:46,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:47,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:49,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:51,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:51,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:51,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:51,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:51,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:12:51,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:55,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:12:58,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:12:58,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:12:58,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:12:58,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:12:58,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:12:58,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:12:58,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:12:58,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:12:59,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:01,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:06,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:06,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:06,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:06,062 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:07,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:07,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:08,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:08,876 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:13:11,318 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:11,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:11,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:11,324 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:12,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:12,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:12,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:13,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:13,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:14,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:16,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:16,437 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:16,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:16,442 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:17,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:17,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:18,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:22,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:23,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:23,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:23,810 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:23,815 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:24,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:26,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:27,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:28,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:28,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:28,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:28,688 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:28,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:28,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:28,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:29,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:30,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:33,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:35,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:35,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:35,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:35,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:36,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:36,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:38,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:38,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:13:39,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:39,812 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:39,812 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:39,817 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:40,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:42,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:43,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:43,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:43,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:43,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:13:44,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:44,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:44,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:44,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:45,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:46,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:49,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:49,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:49,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:49,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:49,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:50,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:52,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:54,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:13:55,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:13:55,849 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:13:55,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:13:55,854 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:13:56,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:13:58,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:13:58,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:13:58,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:13:59,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:00,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:02,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:02,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:02,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:02,393 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:03,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:04,114 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:05,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:08,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:14:09,105 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:09,105 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:09,105 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:09,110 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:09,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:10,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:11,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:13,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:13,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:14,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:16,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:16,091 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:16,092 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:16,097 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:16,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:16,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:18,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:21,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:21,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:21,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:21,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:21,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:22,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:22,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:25,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:25,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:25,370 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:25,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:26,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:26,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:27,375 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:28,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:28,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:29,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:31,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:31,132 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:31,133 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:31,146 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:31,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:32,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:33,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:35,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:35,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:35,165 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:35,170 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:36,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:36,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:38,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:14:38,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:39,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:39,461 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:39,461 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:39,466 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:40,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:40,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:43,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:43,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:43,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:14:43,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:45,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:45,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:45,784 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:45,789 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:46,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:48,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:49,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:51,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:51,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:51,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:51,105 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:51,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:52,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:55,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:14:56,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:14:56,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:14:56,035 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:14:56,040 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:14:56,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:14:58,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:14:58,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:14:58,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:14:58,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:00,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:00,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:00,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:00,511 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:00,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:01,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:02,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:05,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:05,146 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:05,146 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:05,151 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:05,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:06,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:06,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:08,890 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:15:11,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:12,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:12,282 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:12,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:12,287 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:13,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:14,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:14,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:14,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:14,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:17,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:21,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:21,409 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:21,409 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:21,414 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:22,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:22,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:23,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:25,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:25,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:25,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:25,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:26,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:28,206 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:28,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:29,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:29,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:29,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:30,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:30,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:30,185 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:30,190 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:30,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:32,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:34,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:35,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:35,378 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:35,378 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:35,383 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:36,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:36,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:38,887 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:15:39,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:41,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:41,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:41,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:41,473 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:42,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:42,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:44,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:44,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:15:44,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:45,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:47,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:47,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:47,136 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:47,141 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:47,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:48,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:51,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:52,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:52,486 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:52,486 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:52,491 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:53,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:54,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:56,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:15:57,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:15:57,340 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:15:57,341 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:15:57,346 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:15:58,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:15:58,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:15:59,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:15:59,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:15:59,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:01,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:02,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:02,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:02,555 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:02,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:03,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:04,253 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:06,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:06,621 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:06,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:06,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:07,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:07,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:08,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:08,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:16:11,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:11,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:11,174 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:11,179 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:11,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:12,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:13,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:14,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:14,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:14,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:17,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:17,042 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:17,042 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:17,047 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:17,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:18,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:19,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:21,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:21,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:21,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:21,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:21,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:22,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:24,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:25,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:25,246 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:25,246 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:25,251 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:25,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:26,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:29,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:29,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:29,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:29,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:30,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:30,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:30,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:30,401 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:31,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:32,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:34,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:34,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:34,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:34,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:34,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:35,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:36,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:38,891 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:16:39,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:39,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:39,253 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:39,258 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:39,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:40,258 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:40,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:43,525 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:43,526 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:43,526 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:43,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:44,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:44,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:44,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:44,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:16:45,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:49,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:49,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:49,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:49,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:49,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:50,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:51,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:54,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:16:54,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:16:54,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:16:54,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:16:55,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:16:56,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:16:56,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:16:59,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:16:59,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:16:59,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:00,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:00,457 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:00,457 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:00,462 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:01,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:02,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:02,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:07,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:08,142 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:08,143 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:08,143 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:08,148 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:08,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:08,893 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:17:10,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:12,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:12,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:12,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:12,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:12,750 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:13,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:14,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:14,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:14,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:14,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:18,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:18,248 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:18,248 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:18,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:18,253 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:18,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:20,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:22,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:22,480 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:22,481 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:22,485 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:23,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:23,486 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:24,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:26,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:26,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:26,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:26,744 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:27,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:28,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:28,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:29,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:29,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:29,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:31,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:31,598 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:31,598 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:31,603 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:32,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:32,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:34,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:36,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:36,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:36,115 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:36,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:36,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:38,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:38,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:17:39,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:40,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:40,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:40,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:40,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:41,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:42,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:44,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:45,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:45,508 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:45,508 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:45,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:45,513 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:46,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:46,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:49,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:49,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:49,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:49,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:50,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:50,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:50,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:54,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:54,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:54,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:54,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:17:55,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:17:56,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:17:56,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:17:59,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:17:59,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:17:59,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:17:59,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:17:59,913 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:17:59,913 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:17:59,918 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:00,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:00,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:01,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:06,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:06,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:06,179 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:06,184 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:06,452 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:07,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:08,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:08,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:18:11,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:11,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:11,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:11,661 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:12,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:12,486 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:12,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:14,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:14,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:14,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:15,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:15,504 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:15,504 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:15,509 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:16,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:16,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:18,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:20,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:20,122 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:20,122 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:20,129 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:20,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:22,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:24,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:25,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:25,481 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:25,481 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:25,486 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:25,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:26,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:29,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:29,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:29,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:29,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:30,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:30,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:30,115 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:30,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:30,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:32,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:34,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:34,630 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:34,630 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:34,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:34,635 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:35,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:36,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:38,899 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:18:39,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:41,842 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:41,846 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:41,846 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:41,851 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:42,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:42,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:18:44,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:45,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:47,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:47,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:47,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:47,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:48,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:48,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:50,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:52,233 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:18:52,234 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:18:52,234 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:18:52,239 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:18:52,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:18:54,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:18:56,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:18:59,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:18:59,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:18:59,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:00,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:00,779 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:00,779 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:00,784 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:01,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:01,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:02,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:05,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:05,441 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:05,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:05,446 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:05,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:06,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:07,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:08,902 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:19:10,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:10,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:10,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:10,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:10,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:12,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:13,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:14,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:14,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:14,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:14,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:14,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:14,889 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:14,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:15,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:16,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:18,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:19,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:19,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:19,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:19,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:20,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:20,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:23,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:24,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:24,276 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:24,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:24,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:24,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:26,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:29,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:29,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:29,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:29,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:30,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:30,184 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:30,184 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:30,189 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:30,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:32,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:35,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:36,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:36,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:36,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:36,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:36,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:38,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:38,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:19:40,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:42,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:42,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:42,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:42,312 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:42,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:44,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:19:44,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:46,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:47,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:47,951 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:47,951 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:47,956 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:48,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:48,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:51,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:56,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:19:56,153 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:19:56,153 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:19:56,157 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:19:56,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:19:57,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:19:58,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:19:59,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:19:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:19:59,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:01,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:01,886 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:01,886 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:01,890 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:02,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:02,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:02,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:06,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:06,879 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:06,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:06,884 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:07,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:08,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:08,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:08,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:20:13,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:13,424 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:13,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:13,430 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:13,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:14,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:14,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:14,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:14,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:14,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:17,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:17,732 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:17,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:17,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:17,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:18,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:19,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:24,458 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:24,458 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:24,458 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:24,463 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:24,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:25,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:26,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:28,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:28,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:28,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:28,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:28,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:29,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:29,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:29,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:30,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:30,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:35,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:35,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:35,890 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:35,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:35,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:36,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:36,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:38,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:20:40,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:41,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:41,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:41,722 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:41,727 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:41,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:42,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:44,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:44,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:44,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:20:46,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:46,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:46,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:46,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:46,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:47,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:48,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:51,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:51,149 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:51,149 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:51,154 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:51,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:52,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:52,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:20:57,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:20:58,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:20:58,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:20:58,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:20:58,303 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:20:58,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:20:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:20:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:20:59,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:00,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:02,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:02,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:02,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:02,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:02,758 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:02,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:04,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:07,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:07,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:07,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:07,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:07,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:08,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:08,876 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:08,913 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:21:13,174 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:13,175 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:13,175 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:13,180 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:13,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:14,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:14,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:14,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:14,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:14,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:18,021 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:18,022 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:18,022 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:18,027 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:18,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:18,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:20,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:23,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:23,340 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:23,340 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:23,345 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:23,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:24,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:25,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:28,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:28,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:28,262 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:28,267 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:28,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:29,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:29,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:29,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:30,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:30,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:35,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:36,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:36,728 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:36,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:36,734 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:36,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:38,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:38,912 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:21:40,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:41,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:41,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:41,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:41,030 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:41,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:42,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:44,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:44,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:21:44,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:46,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:48,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:48,526 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:48,527 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:48,532 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:48,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:50,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:52,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:52,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:52,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:52,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:52,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:52,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:54,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:57,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:21:57,574 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:21:57,574 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:21:57,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:21:57,579 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:21:57,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:21:58,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:21:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:21:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:21:59,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:02,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:02,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:02,879 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:02,879 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:02,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:02,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:04,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:07,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:07,308 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:07,308 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:07,313 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:07,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:08,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:08,914 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:22:08,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:13,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:14,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:14,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:14,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:14,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:14,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:14,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:14,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:14,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:14,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:18,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:18,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:18,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:18,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:18,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:19,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:20,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:23,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:23,193 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:23,194 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:23,199 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:23,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:24,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:25,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:27,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:27,794 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:27,794 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:27,799 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:27,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:28,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:29,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:29,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:29,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:30,723 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:32,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:32,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:32,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:32,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:32,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:34,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:36,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:38,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:22:39,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:39,801 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:39,801 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:39,806 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:39,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:40,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:41,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:43,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:43,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:43,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:43,659 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:43,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:44,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:44,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:22:44,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:44,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:47,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:48,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:48,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:48,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:48,717 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:48,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:50,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:52,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:56,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:22:56,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:22:56,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:22:56,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:22:56,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:22:56,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:22:58,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:22:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:22:59,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:22:59,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:00,559 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:00,560 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:00,560 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:00,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:00,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:02,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:03,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:05,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:05,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:05,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:05,974 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:05,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:06,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:08,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:23:08,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:10,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:10,817 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:10,818 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:10,823 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:11,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:13,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:14,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:14,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:14,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:14,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:17,294 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:17,294 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:17,295 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:17,300 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:18,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:19,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:20,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:23,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:23,308 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:23,308 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:23,313 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:24,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:25,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:26,314 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:29,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:29,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:29,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:31,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:31,553 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:31,553 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:31,554 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:31,558 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:32,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:33,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:35,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:35,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:35,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:35,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:36,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:36,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:38,924 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:23:39,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:40,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:40,972 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:40,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:40,977 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:41,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:41,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:43,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:44,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:44,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:44,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:23:45,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:45,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:45,991 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:45,997 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:46,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:47,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:49,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:50,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:50,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:50,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:50,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:51,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:52,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:53,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:55,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:23:55,548 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:23:55,548 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:23:55,553 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:23:56,073 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:23:57,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:23:57,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:23:59,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:23:59,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:23:59,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:01,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:01,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:01,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:01,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:02,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:02,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:03,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:05,765 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:05,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:05,766 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:05,771 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:06,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:07,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:07,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:08,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:24:11,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:11,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:11,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:11,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:12,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:13,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:13,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:14,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:14,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:14,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:15,819 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:15,819 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:15,819 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:15,824 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:16,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:17,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:18,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:19,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:19,748 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:19,748 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:19,754 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:20,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:21,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:24,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:25,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:25,583 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:25,584 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:25,589 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:26,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:27,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:29,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:29,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:29,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:30,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:32,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:32,816 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:32,817 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:32,822 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:33,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:35,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:35,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:36,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:36,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:36,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:36,927 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:37,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:38,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:24:39,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:40,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:44,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:44,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:44,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:24:45,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:45,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:45,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:45,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:45,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:46,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:47,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:50,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:50,282 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:50,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:50,287 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:51,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:51,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:53,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:55,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:24:55,382 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:24:55,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:24:55,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:24:56,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:24:56,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:24:57,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:24:59,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:24:59,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:24:59,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:00,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:00,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:00,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:00,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:00,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:02,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:03,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:04,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:04,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:04,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:04,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:05,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:07,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:07,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:08,927 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:25:09,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:09,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:09,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:09,016 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:09,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:11,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:13,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:14,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:14,103 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:14,103 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:14,108 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:14,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:14,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:14,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:14,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:17,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:18,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:18,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:18,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:18,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:18,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:19,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:21,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:23,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:23,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:23,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:23,453 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:24,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:24,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:25,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:27,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:27,855 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:27,855 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:27,860 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:28,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:29,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:29,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:29,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:29,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:29,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:33,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:33,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:33,126 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:33,131 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:33,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:35,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:35,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:37,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:37,743 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:37,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:37,748 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:38,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:38,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:25:39,206 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:40,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:42,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:42,384 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:42,384 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:42,389 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:43,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:44,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:44,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:44,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:45,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:46,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:46,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:46,891 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:46,892 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:46,897 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:47,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:49,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:51,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:51,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:51,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:51,451 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:25:52,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:25:52,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:53,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:25:57,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:25:59,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:25:59,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:25:59,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:25:59,593 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:25:59,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:25:59,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:25:59,700 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:00,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:01,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:02,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:03,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:03,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:03,833 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:03,838 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:04,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:05,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:07,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:08,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:08,563 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:08,563 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:08,568 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:08,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:26:09,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:11,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:12,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:13,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:13,683 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:13,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:13,689 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:14,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:14,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:14,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:14,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:15,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:18,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:18,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:18,209 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:18,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:18,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:18,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:21,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:23,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:23,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:23,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:23,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:23,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:24,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:25,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:28,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:29,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:29,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:29,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:32,758 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:32,758 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:32,759 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:32,764 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:33,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:33,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:35,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:37,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:37,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:37,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:37,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:38,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:38,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:26:38,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:39,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:43,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:43,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:43,117 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:43,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:43,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:44,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:44,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:44,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:44,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:45,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:47,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:47,926 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:47,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:47,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:48,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:49,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:49,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:53,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:53,812 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:53,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:53,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:26:54,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:26:55,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:26:55,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:26:59,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:26:59,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:26:59,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:26:59,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:26:59,894 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:26:59,894 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:26:59,899 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:00,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:00,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:01,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:05,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:05,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:05,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:05,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:06,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:06,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:07,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:08,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:27:10,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:10,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:10,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:10,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:10,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:11,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:12,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:14,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:14,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:14,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:17,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:17,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:17,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:17,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:17,284 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:17,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:19,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:22,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:23,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:23,568 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:23,568 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:23,573 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:24,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:25,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:27,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:28,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:28,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:28,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:28,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:28,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:29,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:29,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:31,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:32,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:33,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:33,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:33,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:33,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:33,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:35,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:37,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:37,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:37,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:37,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:37,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:38,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:38,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:27:39,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:43,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:43,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:43,268 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:43,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:43,273 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:43,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:44,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:44,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:44,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:27:45,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:47,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:47,728 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:47,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:47,734 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:48,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:48,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:49,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:53,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:55,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:27:55,487 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:27:55,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:27:55,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:27:56,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:27:57,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:27:59,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:27:59,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:27:59,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:27:59,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:00,297 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:00,298 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:00,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:00,303 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:00,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:03,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:05,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:07,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:07,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:07,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:07,376 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:07,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:08,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:28:09,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:10,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:12,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:12,357 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:12,357 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:12,362 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:12,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:14,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:14,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:14,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:15,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:16,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:16,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:16,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:16,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:16,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:17,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:19,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:21,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:21,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:21,098 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:21,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:21,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:22,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:23,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:25,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:25,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:25,518 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:25,523 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:26,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:27,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:27,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:29,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:29,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:29,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:31,525 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:31,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:31,526 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:31,530 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:32,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:32,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:33,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:37,094 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:37,095 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:37,095 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:37,101 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:37,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:38,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:38,942 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:28:39,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:41,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:41,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:41,591 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:41,596 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:42,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:43,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:43,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:44,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:44,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:44,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:47,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:47,603 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:47,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:47,609 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:48,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:48,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:49,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:52,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:52,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:52,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:52,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:53,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:54,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:28:55,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:56,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:28:56,599 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:28:56,599 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:28:56,605 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:28:57,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:28:59,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:28:59,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:28:59,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:28:59,438 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:28:59,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:01,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:01,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:01,511 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:01,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:02,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:03,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:05,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:05,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:05,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:05,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:05,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:06,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:07,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:08,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:29:10,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:10,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:10,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:10,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:10,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:11,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:11,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:14,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:14,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:14,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:14,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:14,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:14,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:14,804 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:15,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:16,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:17,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:19,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:19,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:19,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:19,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:19,461 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:21,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:22,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:25,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:25,089 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:25,089 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:25,094 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:25,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:27,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:28,095 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:29,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:29,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:29,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:30,723 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:30,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:30,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:30,733 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:31,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:33,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:33,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:38,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:38,946 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:29:40,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:40,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:40,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:40,205 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:40,480 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:41,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:44,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:44,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:44,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:44,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:45,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:45,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:45,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:46,486 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:47,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:49,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:50,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:50,939 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:50,939 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:50,944 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:51,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:53,494 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:54,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:29:55,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:55,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:55,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:55,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:29:55,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:29:57,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:29:59,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:29:59,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:29:59,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:29:59,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:29:59,841 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:29:59,841 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:29:59,846 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:00,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:00,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:01,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:03,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:03,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:03,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:03,973 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:04,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:05,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:05,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:08,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:08,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:08,828 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:08,833 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:08,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:30:09,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:11,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:11,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:13,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:13,282 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:13,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:13,288 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:13,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:14,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:14,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:14,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:15,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:17,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:19,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:19,726 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:19,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:19,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:20,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:21,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:22,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:25,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:25,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:25,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:25,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:26,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:27,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:27,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:29,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:29,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:30,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:30,272 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:30,272 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:30,277 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:30,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:31,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:33,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:34,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:34,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:34,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:34,972 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:35,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:37,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:38,965 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:30:38,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:41,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:41,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:41,878 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:41,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:42,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:43,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:44,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:44,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:44,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:44,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:49,635 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:52,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:52,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:52,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:52,429 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:52,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:55,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:30:55,557 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:30:56,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:30:56,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:30:56,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:30:56,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:30:57,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:30:59,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:30:59,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:30:59,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:30:59,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:00,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:05,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:05,146 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:05,146 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:05,151 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:05,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:06,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:07,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:08,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:31:11,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:11,055 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:11,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:11,060 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:11,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:12,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:13,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:14,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:14,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:14,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:17,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:19,449 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:19,450 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:19,450 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:19,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:19,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:21,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:23,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:25,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:25,913 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:25,914 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:25,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:26,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:27,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:28,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:29,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:29,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:30,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:30,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:30,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:30,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:31,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:33,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:33,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:37,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:37,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:37,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:37,429 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:37,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:38,954 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:31:38,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:39,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:42,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:42,313 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:42,313 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:42,319 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:42,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:43,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:44,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:44,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:44,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:44,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:48,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:48,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:48,451 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:48,457 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:48,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:49,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:51,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:54,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:31:56,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:31:56,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:31:56,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:31:56,602 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:31:56,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:31:59,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:31:59,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:31:59,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:31:59,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:31:59,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:04,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:05,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:05,912 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:05,912 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:05,917 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:06,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:07,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:08,956 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:32:09,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:11,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:11,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:11,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:11,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:11,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:13,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:14,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:14,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:14,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:15,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:17,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:17,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:17,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:17,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:18,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:19,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:20,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:24,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:24,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:24,384 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:24,389 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:24,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:25,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:26,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:28,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:28,790 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:28,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:28,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:29,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:29,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:29,650 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:31,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:31,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:33,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:33,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:33,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:33,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:33,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:35,660 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:37,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:37,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:37,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:37,987 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:37,992 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:38,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:38,958 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:32:39,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:42,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:43,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:43,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:43,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:43,137 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:43,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:44,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:44,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:44,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:45,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:47,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:47,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:47,810 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:47,815 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:48,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:48,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:49,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:52,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:52,764 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:52,764 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:52,769 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:32:53,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:32:54,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:32:55,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:32:59,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:32:59,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:32:59,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:32:59,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:32:59,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:32:59,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:32:59,691 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:00,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:01,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:05,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:06,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:06,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:06,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:06,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:06,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:07,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:08,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:33:10,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:11,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:11,851 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:11,851 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:11,856 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:12,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:13,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:14,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:14,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:14,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:16,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:17,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:17,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:17,723 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:17,728 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:18,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:19,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:21,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:22,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:22,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:22,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:22,031 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:22,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:23,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:26,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:26,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:26,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:26,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:26,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:27,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:27,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:29,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:29,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:32,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:32,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:32,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:32,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:32,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:33,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:33,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:37,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:37,399 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:37,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:37,404 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:37,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:38,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:38,963 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:33:39,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:42,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:42,773 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:42,773 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:42,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:43,730 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:43,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:44,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:44,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:44,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:45,734 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:47,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:47,464 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:47,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:47,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:47,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:49,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:49,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:54,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:33:54,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:33:54,847 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:33:54,847 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:33:54,852 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:33:55,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:33:57,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:33:59,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:33:59,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:33:59,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:33:59,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:02,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:02,711 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:02,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:02,716 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:02,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:04,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:05,768 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:06,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:06,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:06,831 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:06,836 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:07,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:08,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:34:09,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:09,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:13,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:13,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:13,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:13,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:13,796 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:14,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:14,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:14,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:15,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:15,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:18,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:18,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:18,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:18,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:18,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:19,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:21,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:23,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:23,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:23,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:23,698 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:23,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:25,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:26,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:28,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:28,405 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:28,405 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:28,410 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:28,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:29,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:29,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:29,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:32,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:33,405 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:33,405 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:33,405 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:33,411 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:33,818 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:35,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:38,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:38,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:34:39,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:39,382 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:39,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:39,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:39,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:41,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:44,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:44,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:44,351 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:44,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:44,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:44,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:44,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:44,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:44,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:45,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:49,270 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:49,271 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:49,271 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:49,276 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:49,836 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:50,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:51,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:34:55,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:34:58,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:34:58,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:34:58,029 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:34:58,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:34:58,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:34:59,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:34:59,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:34:59,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:34:59,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:00,635 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:04,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:04,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:04,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:04,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:04,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:05,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:06,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:08,729 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:08,729 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:08,730 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:08,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:08,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:08,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:35:11,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:11,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:14,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:14,075 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:14,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:14,080 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:14,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:14,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:14,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:14,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:15,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:17,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:22,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:23,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:23,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:23,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:23,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:23,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:25,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:27,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:27,486 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:27,486 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:27,491 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:27,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:28,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:29,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:29,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:29,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:29,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:32,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:32,082 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:32,082 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:32,087 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:32,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:33,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:34,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:38,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:35:39,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:40,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:40,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:40,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:40,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:40,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:41,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:44,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:44,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:44,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:45,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:46,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:46,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:46,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:46,322 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:46,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:47,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:51,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:54,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:54,899 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:54,899 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:54,905 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:55,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:35:56,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:35:57,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:35:59,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:35:59,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:35:59,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:35:59,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:35:59,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:35:59,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:35:59,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:35:59,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:01,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:02,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:04,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:04,340 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:04,340 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:04,345 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:04,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:05,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:08,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:08,973 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:36:09,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:09,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:09,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:09,077 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:09,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:11,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:14,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:14,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:14,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:14,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:15,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:15,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:15,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:15,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:15,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:17,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:19,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:19,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:19,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:19,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:19,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:20,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:21,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:24,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:24,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:24,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:24,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:24,603 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:24,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:25,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:28,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:28,880 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:28,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:28,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:28,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:29,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:29,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:29,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:31,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:34,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:34,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:34,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:34,990 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:34,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:35,943 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:37,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:38,978 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:36:39,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:40,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:40,918 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:40,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:40,924 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:40,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:43,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:44,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:44,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:44,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:46,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:46,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:46,329 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:46,334 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:46,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:47,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:51,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:51,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:51,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:51,254 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:51,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:51,966 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:36:53,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:36:56,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:36:59,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:36:59,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:36:59,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:36:59,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:36:59,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:36:59,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:36:59,657 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:36:59,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:01,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:01,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:03,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:03,825 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:03,825 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:03,830 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:03,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:05,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:06,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:08,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:37:11,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:11,659 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:11,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:11,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:11,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:12,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:13,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:14,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:14,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:14,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:15,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:15,961 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:15,961 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:15,966 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:15,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:17,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:18,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:22,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:22,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:22,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:22,701 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:23,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:23,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:24,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:28,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:28,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:28,145 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:28,150 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:29,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:29,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:29,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:29,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:29,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:30,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:32,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:32,583 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:32,583 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:32,588 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:33,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:34,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:34,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:36,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:36,355 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:36,355 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:36,360 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:37,015 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:38,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:38,981 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:37:39,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:41,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:41,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:41,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:41,421 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:42,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:44,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:44,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:44,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:44,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:37:45,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:49,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:49,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:49,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:49,947 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:50,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:50,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:52,031 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:55,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:37:55,628 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:37:55,628 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:37:55,633 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:37:56,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:37:56,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:37:58,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:37:59,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:37:59,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:37:59,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:00,294 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:00,295 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:00,295 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:00,300 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:01,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:02,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:02,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:04,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:04,973 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:04,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:04,978 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:05,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:07,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:08,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:08,981 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:38:11,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:11,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:11,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:11,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:12,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:13,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:14,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:14,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:14,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:14,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:17,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:17,453 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:17,453 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:17,458 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:18,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:19,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:20,100 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:23,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:23,880 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:23,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:23,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:24,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:24,886 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:26,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:29,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:29,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:29,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:30,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:31,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:31,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:31,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:31,063 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:31,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:34,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:36,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:36,491 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:36,492 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:36,492 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:36,497 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:37,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:38,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:38,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:38:41,594 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:41,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:41,595 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:41,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:41,600 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:42,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:44,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:44,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:44,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:44,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:38:45,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:45,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:45,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:45,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:46,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:46,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:48,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:50,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:50,531 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:50,531 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:50,536 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:51,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:52,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:38:52,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:57,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:38:57,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:38:57,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:38:57,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:38:57,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:38:58,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:38:59,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:38:59,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:38:59,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:00,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:02,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:04,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:04,191 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:04,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:04,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:05,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:06,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:08,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:08,989 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:39:09,826 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:09,827 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:09,827 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:09,832 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:10,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:12,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:13,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:14,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:14,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:16,635 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:16,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:16,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:16,651 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:17,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:18,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:19,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:21,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:21,270 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:21,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:21,276 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:22,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:24,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:25,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:27,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:27,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:27,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:27,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:28,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:29,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:29,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:29,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:30,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:35,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:36,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:36,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:36,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:36,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:37,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:38,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:38,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:39:40,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:41,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:41,926 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:41,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:41,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:42,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:44,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:44,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:44,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:44,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:39:46,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:51,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:39:51,132 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:39:51,132 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:39:51,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:39:51,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:39:52,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:54,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:39:57,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:39:59,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:39:59,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:39:59,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:02,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:02,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:02,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:02,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:02,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:03,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:04,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:07,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:08,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:40:09,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:09,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:09,118 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:09,123 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:09,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:12,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:13,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:14,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:14,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:14,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:16,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:16,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:16,273 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:16,278 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:16,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:18,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:18,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:20,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:20,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:20,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:20,724 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:21,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:22,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:23,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:28,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:28,454 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:28,454 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:28,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:29,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:29,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:29,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:29,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:29,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:30,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:32,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:32,912 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:32,912 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:32,917 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:33,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:34,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:34,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:37,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:37,389 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:37,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:37,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:38,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:38,992 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:40:39,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:40,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:44,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:44,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:44,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:40:45,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:45,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:45,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:45,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:45,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:46,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:48,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:49,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:49,941 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:49,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:49,946 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:50,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:50,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:52,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:54,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:40:54,814 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:40:54,814 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:40:54,820 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:40:55,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:40:56,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:40:56,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:40:59,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:40:59,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:40:59,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:00,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:00,717 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:00,717 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:00,722 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:01,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:02,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:02,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:05,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:05,449 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:05,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:05,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:06,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:08,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:08,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:08,994 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:41:11,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:11,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:11,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:11,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:12,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:13,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:14,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:14,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:14,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:14,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:16,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:16,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:16,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:16,374 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:17,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:18,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:19,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:21,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:21,409 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:21,410 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:21,415 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:22,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:24,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:24,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:26,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:26,455 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:26,455 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:26,460 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:27,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:28,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:29,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:29,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:29,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:29,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:31,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:31,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:31,889 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:31,894 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:32,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:34,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:34,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:36,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:36,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:36,883 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:36,888 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:37,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:38,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:39,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:41:40,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:44,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:44,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:44,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:41:44,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:44,726 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:44,726 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:44,731 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:45,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:45,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:46,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:49,534 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:49,535 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:49,535 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:49,541 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:50,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:51,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:52,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:55,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:41:55,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:41:55,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:41:56,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:41:56,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:41:57,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:41:58,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:41:59,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:41:59,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:41:59,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:00,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:00,485 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:00,485 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:00,490 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:01,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:02,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:02,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:06,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:06,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:06,281 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:06,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:06,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:08,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:08,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:08,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:42:13,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:13,407 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:13,407 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:13,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:13,412 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:13,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:14,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:14,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:14,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:16,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:17,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:17,633 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:17,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:17,638 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:18,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:18,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:20,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:23,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:24,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:24,601 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:24,601 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:24,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:25,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:26,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:29,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:29,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:29,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:29,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:29,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:29,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:29,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:29,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:30,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:32,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:34,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:34,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:34,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:34,136 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:34,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:35,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:36,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:38,585 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:38,586 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:38,586 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:38,591 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:39,004 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:42:39,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:40,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:41,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:42,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:42,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:42,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:42,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:43,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:44,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:44,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:44,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:42:44,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:46,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:47,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:47,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:47,598 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:47,603 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:48,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:50,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:51,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:52,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:52,408 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:52,408 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:52,413 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:52,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:54,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:42:57,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:42:57,362 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:42:57,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:42:57,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:42:57,368 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:42:57,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:42:59,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:42:59,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:42:59,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:00,557 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:01,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:01,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:01,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:01,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:02,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:02,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:04,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:06,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:06,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:06,884 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:06,889 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:07,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:08,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:08,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:09,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:43:14,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:14,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:14,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:14,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:15,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:15,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:15,990 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:15,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:16,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:18,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:19,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:20,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:20,454 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:20,454 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:20,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:20,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:22,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:24,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:26,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:26,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:26,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:26,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:26,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:28,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:29,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:29,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:29,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:29,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:32,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:32,118 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:32,118 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:32,123 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:32,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:34,592 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:35,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:37,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:37,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:37,194 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:37,199 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:37,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:38,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:39,005 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:43:41,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:41,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:41,892 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:41,892 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:41,897 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:42,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:44,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:44,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:44,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:43:44,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:46,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:46,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:46,600 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:46,601 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:46,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:47,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:47,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:51,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:52,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:52,016 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:52,016 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:52,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:52,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:53,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:56,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:43:56,839 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:43:56,839 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:43:56,839 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:43:56,844 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:43:57,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:43:57,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:43:59,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:43:59,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:43:59,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:01,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:01,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:01,353 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:01,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:01,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:02,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:03,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:07,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:09,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:44:09,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:09,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:09,505 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:09,511 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:09,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:11,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:12,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:14,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:14,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:15,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:15,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:15,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:16,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:17,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:17,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:20,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:20,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:20,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:20,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:20,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:21,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:23,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:27,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:27,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:27,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:27,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:28,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:28,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:29,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:29,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:29,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:32,258 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:32,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:32,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:32,264 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:32,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:33,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:34,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:39,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:44:40,010 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:40,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:40,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:40,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:40,679 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:41,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:42,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:44,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:44,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:44,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:44:45,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:46,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:46,271 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:46,271 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:46,277 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:46,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:48,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:51,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:51,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:51,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:51,609 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:51,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:51,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:54,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:55,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:44:55,999 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:44:55,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:44:56,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:44:56,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:44:57,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:44:58,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:44:59,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:44:59,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:44:59,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:02,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:03,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:03,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:03,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:03,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:03,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:03,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:08,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:08,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:08,856 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:08,856 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:08,861 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:09,017 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:45:09,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:09,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:14,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:14,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:14,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:14,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:14,632 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:14,637 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:14,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:15,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:19,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:19,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:19,657 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:19,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:19,662 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:19,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:21,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:24,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:25,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:25,800 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:25,801 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:25,806 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:26,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:27,718 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:29,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:29,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:29,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:30,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:30,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:30,982 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:30,982 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:30,987 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:31,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:31,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:36,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:38,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:38,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:38,331 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:38,336 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:38,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:39,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:45:39,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:42,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:42,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:42,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:42,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:42,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:43,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:43,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:44,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:44,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:44,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:45:47,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:47,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:47,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:47,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:47,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:47,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:47,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:51,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:45:51,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:45:51,253 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:45:51,258 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:45:51,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:45:51,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:45:52,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:57,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:45:59,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:45:59,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:45:59,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:00,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:00,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:00,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:00,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:00,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:01,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:02,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:05,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:05,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:05,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:05,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:05,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:07,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:07,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:09,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:46:10,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:10,158 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:10,158 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:10,163 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:10,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:11,790 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:13,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:14,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:14,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:16,033 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:16,033 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:16,033 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:16,039 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:16,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:17,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:19,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:21,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:21,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:21,125 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:21,130 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:21,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:21,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:24,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:25,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:25,618 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:25,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:25,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:25,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:27,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:29,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:29,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:29,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:29,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:29,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:30,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:31,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:34,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:35,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:35,841 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:35,841 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:35,846 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:36,818 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:37,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:39,017 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:46:40,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:44,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:44,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:45,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:45,182 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:45,182 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:45,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:45,187 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:45,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:45,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:49,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:49,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:49,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:49,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:50,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:50,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:51,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:54,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:54,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:54,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:54,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:54,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:46:55,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:56,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:46:59,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:46:59,256 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:46:59,256 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:46:59,262 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:46:59,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:46:59,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:46:59,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:46:59,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:46:59,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:01,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:06,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:06,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:06,018 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:06,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:06,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:07,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:07,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:09,020 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:47:10,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:10,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:10,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:10,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:10,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:11,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:12,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:14,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:14,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:15,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:15,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:15,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:15,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:15,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:17,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:18,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:20,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:20,577 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:20,578 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:20,583 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:20,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:21,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:24,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:26,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:26,650 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:26,650 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:26,655 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:26,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:27,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:29,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:29,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:31,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:31,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:31,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:31,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:31,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:31,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:35,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:38,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:38,699 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:38,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:38,704 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:38,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:39,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:47:39,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:41,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:43,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:43,534 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:43,534 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:43,539 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:43,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:43,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:44,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:44,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:47:44,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:46,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:48,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:48,093 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:48,093 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:48,098 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:48,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:49,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:52,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:52,826 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:47:52,827 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:47:52,827 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:47:52,832 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:47:52,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:47:53,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:47:57,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:47:59,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:47:59,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:47:59,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:00,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:00,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:00,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:00,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:00,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:01,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:03,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:05,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:05,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:05,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:05,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:05,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:05,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:08,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:09,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:48:10,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:10,431 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:10,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:10,436 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:10,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:11,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:13,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:14,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:14,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:14,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:15,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:15,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:15,268 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:15,273 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:15,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:15,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:19,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:19,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:19,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:19,337 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:19,342 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:19,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:20,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:24,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:24,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:24,625 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:24,625 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:24,630 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:25,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:26,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:29,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:29,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:29,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:30,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:30,742 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:30,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:30,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:31,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:32,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:34,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:35,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:35,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:35,210 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:35,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:36,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:36,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:39,029 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:48:40,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:41,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:41,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:41,121 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:41,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:42,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:42,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:44,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:44,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:46,207 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:46,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:46,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:46,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:47,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:48,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:51,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:54,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:54,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:54,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:54,209 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:48:55,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:48:56,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:48:57,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:48:59,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:48:59,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:48:59,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:48:59,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:48:59,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:48:59,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:48:59,721 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:00,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:02,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:02,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:05,729 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:05,730 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:05,730 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:05,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:06,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:07,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:08,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:09,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:49:09,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:09,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:09,929 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:09,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:10,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:12,114 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:12,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:14,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:14,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:14,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:14,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:14,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:14,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:14,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:15,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:16,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:18,518 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:18,519 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:18,519 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:18,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:18,524 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:19,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:20,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:23,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:26,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:26,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:26,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:26,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:27,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:28,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:28,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:29,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:29,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:29,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:32,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:32,825 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:32,825 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:32,830 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:33,134 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:34,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:34,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:39,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:49:40,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:41,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:41,918 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:41,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:41,924 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:42,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:44,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:44,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:49:45,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:46,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:46,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:46,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:46,404 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:47,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:48,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:51,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:52,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:52,077 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:52,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:52,083 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:52,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:54,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:56,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:49:56,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:49:56,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:49:56,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:49:56,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:49:57,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:49:58,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:49:59,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:49:59,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:49:59,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:02,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:02,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:02,355 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:02,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:02,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:03,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:04,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:07,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:07,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:07,464 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:07,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:07,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:08,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:08,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:09,032 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:50:11,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:11,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:11,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:11,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:12,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:12,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:12,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:14,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:14,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:14,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:17,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:17,412 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:17,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:17,417 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:18,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:18,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:18,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:23,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:23,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:23,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:23,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:23,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:24,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:26,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:28,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:28,577 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:28,577 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:28,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:28,583 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:29,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:29,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:29,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:30,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:33,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:35,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:35,536 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:35,536 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:35,541 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:36,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:36,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:39,034 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:50:39,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:43,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:43,838 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:43,838 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:43,843 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:44,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:44,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:44,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:46,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:50:49,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:54,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:50:59,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:50:59,221 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:50:59,221 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:50:59,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:50:59,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:50:59,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:50:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:50:59,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:50:59,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:00,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:04,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:04,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:04,251 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:04,256 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:04,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:05,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:06,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:08,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:08,871 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:08,871 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:08,876 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:09,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:51:09,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:10,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:11,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:13,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:13,887 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:13,888 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:13,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:14,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:14,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:14,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:14,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:16,291 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:16,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:21,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:22,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:22,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:22,975 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:22,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:23,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:24,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:27,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:28,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:28,844 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:28,845 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:28,850 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:29,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:29,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:29,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:29,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:30,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:32,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:33,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:33,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:33,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:33,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:33,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:34,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:37,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:37,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:37,404 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:37,409 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:38,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:38,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:38,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:39,038 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:51:42,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:42,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:42,351 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:42,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:43,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:44,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:44,357 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:44,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:44,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:51:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:47,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:47,588 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:47,588 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:47,593 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:48,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:48,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:49,632 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:53,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:53,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:53,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:53,041 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:53,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:54,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:51:55,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:51:58,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:51:58,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:51:58,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:51:58,041 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:51:58,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:51:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:51:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:51:59,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:00,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:00,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:02,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:02,752 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:02,752 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:02,757 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:03,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:04,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:05,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:08,315 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:08,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:08,316 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:08,321 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:08,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:09,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:52:10,350 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:11,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:14,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:14,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:14,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:14,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:14,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:14,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:14,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:15,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:16,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:19,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:19,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:19,751 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:19,756 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:20,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:20,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:21,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:23,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:23,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:23,874 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:23,879 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:24,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:24,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:26,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:29,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:29,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:29,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:29,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:29,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:29,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:29,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:30,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:32,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:37,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:39,043 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:52:39,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:39,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:39,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:39,370 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:39,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:40,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:43,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:44,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:44,393 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:44,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:44,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:44,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:44,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:52:45,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:46,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:48,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:50,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:50,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:50,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:50,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:51,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:52,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:53,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:56,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:52:56,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:52:56,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:52:56,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:52:57,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:52:58,396 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:52:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:52:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:52:59,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:52:59,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:01,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:01,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:01,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:01,119 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:01,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:02,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:05,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:05,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:05,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:05,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:05,758 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:06,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:06,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:09,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:53:11,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:11,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:11,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:11,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:11,104 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:11,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:12,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:14,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:14,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:14,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:16,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:16,624 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:16,624 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:16,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:16,629 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:17,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:18,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:21,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:24,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:24,021 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:24,021 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:24,027 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:24,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:26,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:27,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:29,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:29,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:29,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:29,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:29,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:29,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:29,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:30,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:30,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:32,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:34,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:34,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:34,180 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:34,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:34,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:36,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:38,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:38,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:38,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:38,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:38,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:38,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:39,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:53:40,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:42,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:42,764 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:42,764 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:42,769 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:43,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:43,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:44,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:44,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:44,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:53:44,471 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:48,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:48,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:48,297 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:48,302 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:48,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:49,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:50,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:52,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:52,408 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:52,409 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:52,414 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:52,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:54,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:54,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:57,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:53:57,676 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:53:57,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:53:57,681 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:53:58,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:53:58,497 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:53:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:53:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:53:59,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:53:59,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:04,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:04,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:04,645 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:04,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:04,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:05,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:06,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:09,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:54:10,051 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:10,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:10,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:10,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:10,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:10,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:12,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:14,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:14,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:14,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:14,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:14,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:15,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:15,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:16,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:20,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:20,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:20,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:20,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:20,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:21,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:22,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:24,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:24,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:24,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:24,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:25,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:26,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:26,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:29,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:29,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:29,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:29,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:29,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:29,849 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:29,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:30,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:30,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:31,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:34,405 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:34,406 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:34,406 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:34,411 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:34,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:36,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:37,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:38,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:38,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:38,878 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:38,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:39,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:54:39,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:40,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:43,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:43,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:43,541 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:43,541 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:43,546 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:43,575 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:44,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:44,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:54:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:44,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:48,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:49,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:49,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:49,202 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:49,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:49,582 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:50,585 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:54,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:54,016 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:54,016 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:54,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:54,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:54,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:54,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:57,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:54:57,950 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:54:57,950 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:54:57,955 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:54:58,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:54:58,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:54:59,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:54:59,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:54:59,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:54:59,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:04,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:04,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:04,117 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:04,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:04,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:05,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:06,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:09,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:55:11,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:11,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:11,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:11,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:11,405 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:11,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:12,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:14,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:14,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:16,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:18,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:18,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:18,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:18,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:18,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:20,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:22,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:26,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:26,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:26,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:26,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:27,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:27,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:28,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:29,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:29,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:30,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:30,856 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:30,856 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:30,862 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:31,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:32,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:32,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:35,569 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:35,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:35,570 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:35,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:35,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:36,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:38,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:39,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:55:39,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:39,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:39,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:39,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:39,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:40,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:44,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:44,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:44,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:55:44,723 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:44,723 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:44,724 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:44,729 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:45,663 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:46,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:49,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:49,132 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:49,132 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:49,137 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:49,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:50,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:50,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:53,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:53,607 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:53,607 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:53,612 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:53,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:54,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:55,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:55:57,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:55:57,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:55:57,977 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:55:57,982 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:55:58,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:55:58,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:55:59,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:55:59,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:55:59,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:00,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:02,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:02,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:02,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:02,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:02,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:04,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:06,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:08,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:08,403 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:08,403 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:08,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:08,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:09,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:56:10,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:12,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:14,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:14,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:14,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:14,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:14,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:14,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:14,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:16,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:17,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:18,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:18,536 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:18,536 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:18,541 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:18,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:20,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:23,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:23,197 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:23,197 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:23,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:23,202 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:23,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:24,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:27,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:27,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:27,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:27,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:28,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:28,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:28,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:29,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:29,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:33,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:33,027 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:33,027 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:33,032 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:33,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:34,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:34,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:38,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:38,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:38,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:38,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:38,754 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:39,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:56:40,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:40,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:42,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:42,496 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:42,496 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:42,501 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:42,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:44,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:44,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:44,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:47,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:47,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:47,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:47,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:47,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:48,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:50,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:53,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:53,530 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:53,530 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:53,535 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:53,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:56:54,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:56:56,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:56:59,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:56:59,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:56:59,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:56:59,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:56:59,729 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:56:59,729 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:56:59,734 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:56:59,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:00,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:01,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:04,610 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:04,611 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:04,611 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:04,616 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:04,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:06,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:07,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:09,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:57:11,853 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:11,854 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:11,854 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:11,859 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:12,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:12,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:12,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:14,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:14,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:18,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:18,431 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:18,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:18,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:18,436 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:18,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:20,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:23,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:26,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:26,213 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:26,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:26,218 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:26,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:26,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:29,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:29,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:29,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:34,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:34,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:34,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:34,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:34,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:35,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:36,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:39,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:57:39,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:39,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:39,783 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:39,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:39,787 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:39,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:40,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:44,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:44,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:57:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:47,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:47,339 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:47,340 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:47,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:47,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:48,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:51,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:54,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:54,264 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:54,264 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:54,269 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:54,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:56,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:57:57,270 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:57:58,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:57:58,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:57:58,831 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:57:58,836 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:57:58,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:57:59,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:57:59,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:57:59,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:00,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:02,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:04,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:04,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:04,258 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:04,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:04,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:06,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:08,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:08,123 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:08,123 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:08,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:08,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:08,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:08,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:09,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:58:13,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:13,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:13,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:13,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:13,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:13,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:14,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:14,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:14,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:17,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:17,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:17,572 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:17,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:17,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:18,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:18,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:21,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:21,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:21,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:21,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:21,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:22,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:23,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:26,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:26,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:26,349 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:26,354 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:26,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:28,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:29,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:29,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:29,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:31,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:31,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:31,018 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:31,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:31,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:32,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:35,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:37,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:37,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:37,030 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:37,035 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:38,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:39,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:39,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:58:40,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:41,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:41,567 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:41,567 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:41,572 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:42,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:43,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:44,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:44,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:58:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:46,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:46,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:46,658 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:46,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:47,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:49,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:51,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:53,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:53,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:53,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:53,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:54,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:55,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:57,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:58:58,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:58:58,144 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:58:58,144 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:58:58,149 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:58:59,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:58:59,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:58:59,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:58:59,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:58:59,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:02,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:03,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:03,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:03,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:03,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:04,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:05,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:08,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:08,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:08,993 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:08,994 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:08,999 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:09,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:09,071 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:59:11,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:13,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:13,906 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:13,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:13,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:13,911 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:14,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:14,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:14,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:15,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:18,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:18,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:18,379 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:18,384 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:19,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:19,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:21,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:24,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:26,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:26,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:26,991 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:26,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:27,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:29,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:29,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:29,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:29,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:29,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:32,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:32,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:32,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:32,771 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:33,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:34,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:35,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:37,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:37,164 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:37,164 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:37,169 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:38,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:39,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:39,074 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 12:59:40,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:42,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:42,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:42,463 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:42,468 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:43,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:44,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:44,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 12:59:45,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:45,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:47,286 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:47,287 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:47,287 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:47,292 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:48,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:49,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:51,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:54,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:54,085 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:54,085 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:54,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:55,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:55,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:57,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 12:59:58,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 12:59:58,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 12:59:58,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 12:59:58,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 12:59:59,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 12:59:59,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 12:59:59,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 12:59:59,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 12:59:59,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:02,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:02,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:02,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:02,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:02,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:03,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:03,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:07,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:09,076 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:00:10,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:10,803 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:10,803 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:10,808 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:11,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:12,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:13,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:14,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:14,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:14,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:16,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:16,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:16,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:16,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:17,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:17,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:18,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:22,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:22,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:22,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:22,649 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:23,107 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:23,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:25,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:27,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:27,077 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:27,077 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:27,082 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:27,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:29,083 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:29,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:29,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:29,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:29,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:32,142 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:32,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:32,143 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:32,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:33,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:33,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:34,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:39,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:00:39,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:39,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:39,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:39,114 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:39,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:40,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:41,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:44,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:44,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:44,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:00:44,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:44,773 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:44,773 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:44,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:45,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:45,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:47,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:50,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:50,042 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:50,043 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:50,048 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:50,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:51,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:51,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:56,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:00:58,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:00:58,414 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:00:58,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:00:58,420 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:00:59,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:00:59,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:00:59,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:00:59,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:00:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:01,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:02,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:02,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:02,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:02,870 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:03,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:05,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:06,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:06,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:06,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:06,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:06,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:07,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:09,080 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:01:09,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:12,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:12,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:12,118 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:12,119 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:12,124 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:12,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:13,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:14,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:14,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:16,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:16,419 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:16,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:16,424 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:17,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:17,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:17,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:21,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:21,702 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:21,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:21,707 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:22,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:22,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:23,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:27,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:29,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:29,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:30,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:30,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:30,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:30,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:31,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:31,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:33,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:34,561 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:34,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:34,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:34,567 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:35,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:37,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:38,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:39,082 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:01:39,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:39,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:39,215 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:39,220 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:39,225 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:41,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:44,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:44,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:44,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:01:48,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:48,890 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:48,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:48,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:49,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:49,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:51,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:54,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:54,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:54,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:54,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:54,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:55,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:01:55,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:01:59,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:01:59,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:01:59,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:01:59,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:01:59,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:01:59,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:01:59,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:01:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:00,631 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:01,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:04,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:04,989 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:04,989 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:04,994 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:05,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:06,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:07,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:09,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:02:10,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:10,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:10,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:10,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:11,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:11,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:13,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:14,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:14,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:14,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:17,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:17,606 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:17,606 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:17,607 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:17,612 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:18,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:19,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:22,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:22,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:22,673 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:22,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:22,678 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:23,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:25,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:27,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:27,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:27,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:27,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:27,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:28,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:29,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:29,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:29,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:32,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:33,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:33,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:33,193 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:33,198 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:33,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:35,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:38,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:39,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:02:41,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:41,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:41,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:41,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:42,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:43,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:43,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:44,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:44,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:02:47,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:47,211 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:47,211 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:47,216 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:47,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:49,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:49,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:52,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:52,786 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:52,786 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:52,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:53,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:54,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:02:55,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:58,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:02:58,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:02:58,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:02:58,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:02:58,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:02:59,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:02:59,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:02:59,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:02:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:00,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:02,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:02,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:02,888 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:02,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:03,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:05,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:05,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:09,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:03:11,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:12,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:12,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:12,818 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:12,823 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:13,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:14,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:14,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:15,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:16,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:21,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:21,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:21,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:21,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:21,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:22,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:23,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:26,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:26,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:26,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:26,473 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:27,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:27,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:27,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:29,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:29,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:32,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:33,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:33,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:33,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:33,050 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:33,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:35,396 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:38,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:38,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:38,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:38,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:38,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:38,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:39,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:03:39,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:43,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:44,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:44,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:44,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:44,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:44,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:44,780 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:44,786 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:45,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:47,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:48,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:49,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:49,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:49,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:49,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:49,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:51,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:53,631 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:53,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:53,632 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:53,637 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:54,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:54,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:03:55,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:58,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:03:58,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:03:58,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:03:58,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:03:59,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:03:59,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:03:59,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:03:59,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:03:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:03:59,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:02,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:02,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:02,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:02,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:03,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:04,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:05,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:07,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:07,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:07,811 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:07,816 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:08,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:09,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:04:09,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:10,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:14,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:14,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:15,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:17,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:17,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:17,544 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:17,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:18,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:19,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:21,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:22,843 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:22,843 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:22,844 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:22,849 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:23,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:25,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:26,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:29,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:29,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:29,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:29,639 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:29,639 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:29,644 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:30,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:31,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:32,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:34,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:34,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:34,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:34,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:35,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:37,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:37,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:39,095 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:04:39,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:39,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:39,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:39,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:40,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:41,480 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:43,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:44,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:44,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:44,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:44,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:44,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:44,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:44,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:45,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:48,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:48,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:48,581 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:48,586 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:49,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:49,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:49,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:54,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:04:54,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:04:54,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:04:54,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:04:54,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:04:55,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:04:55,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:04:59,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:04:59,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:04:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:04:59,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:00,581 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:00,582 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:00,582 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:00,588 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:01,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:01,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:05,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:05,866 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:05,866 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:05,867 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:05,872 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:06,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:07,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:09,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:05:11,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:11,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:11,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:11,609 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:11,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:12,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:13,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:14,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:14,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:16,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:16,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:16,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:16,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:17,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:19,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:21,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:21,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:21,209 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:21,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:21,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:22,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:23,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:25,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:25,378 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:25,378 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:25,383 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:25,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:27,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:27,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:29,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:29,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:29,281 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:29,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:29,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:29,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:29,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:31,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:32,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:33,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:33,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:33,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:34,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:34,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:35,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:38,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:39,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:05:40,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:40,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:40,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:40,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:40,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:41,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:43,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:44,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:44,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:05:45,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:45,441 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:45,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:45,446 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:45,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:47,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:48,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:49,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:49,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:49,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:49,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:50,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:51,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:53,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:54,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:05:54,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:05:54,789 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:05:54,794 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:05:55,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:05:55,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:05:59,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:05:59,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:05:59,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:05:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:00,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:00,089 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:00,089 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:00,094 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:00,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:01,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:04,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:04,540 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:04,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:04,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:04,546 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:04,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:05,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:08,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:08,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:08,964 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:08,969 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:09,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:06:09,575 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:10,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:11,582 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:13,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:13,247 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:13,248 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:13,253 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:13,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:14,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:14,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:15,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:15,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:18,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:18,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:18,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:18,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:19,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:20,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:21,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:23,750 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:23,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:23,751 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:23,756 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:24,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:25,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:26,757 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:29,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:29,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:31,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:31,786 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:31,786 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:31,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:31,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:32,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:33,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:36,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:36,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:36,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:36,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:36,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:37,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:37,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:39,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:06:40,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:40,508 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:40,509 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:40,514 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:40,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:41,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:42,514 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:44,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:44,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:06:45,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:45,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:45,744 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:45,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:46,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:47,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:47,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:49,765 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:49,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:49,766 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:49,771 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:50,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:51,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:53,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:54,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:06:54,552 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:06:54,553 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:06:54,558 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:06:54,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:06:55,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:06:59,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:06:59,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:06:59,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:06:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:00,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:00,826 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:00,826 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:00,831 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:01,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:01,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:04,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:05,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:05,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:05,343 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:05,348 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:05,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:07,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:09,105 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:07:09,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:09,963 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:09,963 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:09,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:09,968 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:10,646 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:11,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:14,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:14,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:15,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:15,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:15,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:15,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:15,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:15,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:17,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:19,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:19,270 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:19,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:19,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:19,656 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:20,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:21,660 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:24,581 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:24,582 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:24,582 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:24,587 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:24,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:25,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:25,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:29,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:29,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:29,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:29,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:29,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:29,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:30,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:30,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:31,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:35,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:36,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:36,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:36,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:36,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:36,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:37,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:39,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:07:40,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:40,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:40,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:40,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:40,804 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:41,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:41,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:44,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:44,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:07:46,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:47,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:47,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:47,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:47,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:48,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:49,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:51,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:52,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:52,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:52,353 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:52,358 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:52,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:53,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:56,569 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:07:56,570 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:07:56,570 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:07:56,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:07:56,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:07:57,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:07:57,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:07:59,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:07:59,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:07:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:01,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:01,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:01,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:01,308 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:01,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:03,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:03,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:06,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:06,790 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:06,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:06,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:07,749 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:07,749 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:08,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:09,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:08:11,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:11,016 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:11,016 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:11,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:11,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:13,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:14,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:14,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:14,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:17,458 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:17,458 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:17,459 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:17,464 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:17,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:19,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:19,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:21,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:21,728 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:21,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:21,733 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:21,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:23,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:24,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:25,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:25,955 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:25,955 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:25,960 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:26,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:27,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:29,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:29,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:29,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:30,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:30,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:30,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:30,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:31,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:33,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:35,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:35,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:35,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:35,231 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:35,236 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:35,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:37,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:39,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:08:39,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:39,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:39,278 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:39,283 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:39,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:40,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:41,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:44,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:44,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:44,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:44,419 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:44,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:44,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:44,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:45,759 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:45,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:50,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:51,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:51,574 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:51,574 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:51,579 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:51,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:53,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:08:56,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:08:58,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:08:58,513 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:08:58,513 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:08:58,518 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:08:58,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:08:59,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:08:59,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:08:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:08:59,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:01,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:02,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:02,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:02,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:02,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:03,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:03,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:06,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:07,837 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:07,838 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:07,838 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:07,843 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:08,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:09,113 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:09:09,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:12,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:13,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:13,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:13,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:13,401 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:13,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:14,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:14,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:15,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:17,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:19,311 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:19,312 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:19,312 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:19,317 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:19,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:21,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:23,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:25,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:25,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:25,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:25,298 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:25,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:27,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:29,299 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:29,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:29,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:31,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:31,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:31,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:31,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:31,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:33,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:35,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:37,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:37,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:37,584 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:37,589 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:37,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:39,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:09:39,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:41,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:43,090 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:43,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:43,091 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:43,096 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:43,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:44,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:44,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:45,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:46,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:47,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:47,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:47,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:47,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:47,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:49,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:51,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:51,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:51,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:51,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:51,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:52,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:53,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:09:56,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:09:57,765 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:09:57,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:09:57,766 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:09:57,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:09:57,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:09:59,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:09:59,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:09:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:09:59,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:02,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:02,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:02,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:02,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:02,074 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:02,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:03,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:06,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:06,153 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:06,154 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:06,159 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:06,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:07,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:07,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:09,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:10:10,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:10,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:10,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:10,687 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:10,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:11,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:12,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:14,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:14,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:14,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:14,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:14,779 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:14,779 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:14,786 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:14,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:15,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:17,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:22,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:22,814 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:22,814 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:22,819 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:22,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:23,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:23,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:28,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:28,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:28,857 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:28,858 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:28,863 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:28,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:29,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:29,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:29,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:29,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:33,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:33,349 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:33,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:33,355 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:33,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:34,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:35,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:38,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:38,589 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:38,590 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:38,595 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:38,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:39,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:10:39,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:40,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:42,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:42,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:42,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:42,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:42,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:43,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:44,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:44,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:44,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:45,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:48,565 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:48,565 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:48,566 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:48,570 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:48,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:49,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:51,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:53,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:53,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:53,405 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:53,410 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:53,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:55,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:10:57,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:10:58,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:10:58,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:10:58,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:10:58,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:10:58,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:10:59,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:10:59,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:10:59,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:10:59,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:02,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:04,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:04,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:04,638 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:04,643 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:04,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:05,943 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:07,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:09,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:11:11,298 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:11,299 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:11,299 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:11,304 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:11,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:13,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:13,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:14,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:14,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:16,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:16,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:16,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:16,236 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:16,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:17,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:19,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:20,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:20,533 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:20,534 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:20,538 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:20,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:21,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:24,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:25,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:25,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:25,028 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:25,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:25,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:25,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:29,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:29,133 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:29,133 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:29,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:29,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:29,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:29,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:30,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:30,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:34,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:34,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:34,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:34,619 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:35,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:35,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:36,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:38,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:38,458 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:38,458 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:38,463 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:39,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:39,124 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:11:40,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:41,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:44,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:44,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:11:44,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:44,835 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:44,835 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:44,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:45,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:46,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:46,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:49,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:49,133 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:49,133 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:49,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:50,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:50,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:52,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:53,787 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:53,787 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:53,787 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:53,793 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:54,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:56,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:11:57,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:11:58,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:11:58,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:11:58,218 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:11:58,224 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:11:59,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:11:59,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:11:59,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:11:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:00,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:02,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:02,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:02,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:02,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:02,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:03,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:04,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:07,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:07,962 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:07,962 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:07,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:07,967 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:08,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:09,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:12:10,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:12,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:12,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:12,874 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:12,879 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:13,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:13,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:14,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:14,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:14,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:17,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:17,382 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:17,382 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:17,387 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:18,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:19,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:20,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:23,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:23,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:23,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:23,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:23,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:24,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:25,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:28,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:28,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:28,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:28,910 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:29,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:29,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:29,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:30,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:32,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:32,966 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:32,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:32,971 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:33,107 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:34,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:36,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:38,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:38,168 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:38,168 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:38,173 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:39,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:39,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:12:40,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:41,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:44,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:44,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:12:46,578 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:46,579 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:46,579 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:46,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:46,584 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:47,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:48,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:51,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:52,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:52,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:52,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:52,717 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:53,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:54,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:12:56,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:12:57,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:12:57,551 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:12:57,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:12:57,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:12:58,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:12:59,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:12:59,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:12:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:00,134 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:02,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:02,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:02,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:02,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:02,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:03,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:04,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:07,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:07,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:07,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:07,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:07,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:08,142 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:09,130 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:13:10,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:12,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:12,994 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:12,995 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:12,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:12,999 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:13,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:14,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:14,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:14,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:18,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:21,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:21,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:21,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:21,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:22,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:23,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:24,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:27,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:27,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:27,991 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:27,996 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:28,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:28,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:29,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:29,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:30,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:32,525 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:32,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:32,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:32,530 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:33,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:34,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:34,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:37,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:37,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:37,447 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:37,451 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:38,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:39,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:13:40,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:40,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:44,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:44,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:44,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:44,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:44,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:44,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:45,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:46,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:50,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:51,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:51,402 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:51,402 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:51,407 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:13:52,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:13:54,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:13:56,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:13:59,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:13:59,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:13:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:13:59,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:13:59,817 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:13:59,817 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:13:59,822 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:00,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:01,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:02,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:06,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:08,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:08,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:08,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:08,092 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:08,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:09,134 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:14:10,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:12,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:12,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:12,362 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:12,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:12,368 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:13,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:14,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:14,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:14,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:16,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:16,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:16,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:16,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:17,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:17,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:18,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:21,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:21,211 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:21,211 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:21,216 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:22,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:22,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:23,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:25,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:25,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:25,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:25,739 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:26,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:28,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:28,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:29,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:29,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:33,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:33,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:33,906 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:33,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:33,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:34,225 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:36,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:38,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:39,136 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:14:41,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:41,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:41,232 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:41,237 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:42,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:42,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:44,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:44,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:44,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:14:46,154 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:46,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:46,155 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:46,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:46,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:48,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:50,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:53,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:53,606 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:53,607 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:53,612 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:54,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:55,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:14:56,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:14:58,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:14:58,630 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:14:58,630 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:14:58,635 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:14:59,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:14:59,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:14:59,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:14:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:00,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:00,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:05,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:06,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:06,880 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:06,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:06,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:07,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:08,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:09,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:15:11,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:13,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:13,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:13,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:13,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:14,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:14,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:14,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:16,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:16,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:20,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:20,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:20,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:20,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:20,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:22,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:22,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:24,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:24,906 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:24,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:24,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:25,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:26,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:27,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:29,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:29,335 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:29,335 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:29,340 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:29,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:29,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:30,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:32,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:33,750 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:38,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:38,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:38,499 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:38,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:39,141 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:15:39,142 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:39,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:40,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:42,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:42,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:42,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:42,550 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:43,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:44,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:44,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:44,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:44,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:15:47,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:47,177 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:47,177 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:47,183 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:47,354 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:50,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:50,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:51,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:15:51,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:15:51,884 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:15:51,889 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:15:52,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:15:54,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:15:55,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:15:59,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:15:59,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:15:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:00,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:00,089 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:00,089 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:00,094 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:00,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:01,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:02,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:06,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:07,083 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:07,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:07,084 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:07,089 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:07,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:09,146 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:16:10,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:11,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:11,075 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:11,076 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:11,081 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:11,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:12,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:14,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:14,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:14,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:17,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:19,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:19,540 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:19,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:19,545 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:20,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:22,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:23,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:27,853 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:27,854 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:27,854 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:27,859 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:28,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:28,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:29,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:29,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:30,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:32,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:32,905 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:32,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:32,910 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:33,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:33,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:34,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:38,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:39,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:16:40,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:40,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:40,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:40,220 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:40,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:42,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:44,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:44,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:44,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:44,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:44,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:44,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:44,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:16:44,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:45,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:46,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:49,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:49,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:49,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:49,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:49,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:50,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:52,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:54,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:16:54,481 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:16:54,481 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:16:54,486 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:16:55,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:16:55,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:16:56,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:16:59,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:16:59,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:16:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:00,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:02,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:02,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:02,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:02,358 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:02,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:04,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:06,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:06,607 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:06,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:06,608 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:06,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:07,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:08,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:09,151 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:17:11,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:11,346 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:11,346 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:11,351 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:11,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:12,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:14,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:14,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:14,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:17,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:19,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:19,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:19,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:19,924 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:20,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:22,480 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:22,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:24,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:24,830 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:24,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:24,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:25,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:26,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:28,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:29,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:29,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:30,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:30,088 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:30,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:30,093 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:30,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:32,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:34,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:34,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:34,592 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:34,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:34,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:35,497 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:36,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:39,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:39,087 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:39,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:39,092 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:39,149 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:17:39,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:39,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:40,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:44,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:44,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:44,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:44,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:44,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:44,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:44,714 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:45,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:46,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:48,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:48,811 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:48,811 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:48,819 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:49,511 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:49,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:50,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:54,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:54,749 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:54,749 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:54,754 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:55,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:17:55,754 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:17:56,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:17:58,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:17:58,939 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:17:58,939 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:17:58,944 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:17:59,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:17:59,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:17:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:17:59,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:00,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:01,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:03,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:03,568 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:03,568 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:03,573 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:04,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:06,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:07,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:07,433 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:07,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:07,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:07,439 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:07,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:09,151 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:18:10,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:12,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:12,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:12,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:12,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:12,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:13,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:14,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:14,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:14,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:16,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:16,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:16,324 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:16,329 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:16,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:18,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:19,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:21,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:21,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:21,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:21,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:21,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:22,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:25,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:26,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:26,837 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:26,837 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:26,842 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:27,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:28,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:29,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:29,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:30,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:32,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:32,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:32,219 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:32,224 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:32,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:34,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:36,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:37,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:37,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:37,239 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:37,244 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:37,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:38,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:39,154 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:18:42,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:42,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:42,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:42,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:42,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:43,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:44,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:44,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:44,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:47,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:49,559 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:49,559 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:49,560 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:49,565 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:49,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:52,582 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:53,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:53,610 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:53,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:53,611 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:53,616 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:54,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:18:56,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:18:58,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:18:58,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:18:58,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:18:58,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:18:58,916 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:18:59,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:18:59,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:18:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:18:59,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:00,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:03,724 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:04,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:04,301 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:04,301 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:04,307 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:04,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:06,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:08,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:08,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:08,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:08,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:08,927 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:09,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:19:09,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:10,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:13,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:13,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:13,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:13,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:13,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:14,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:14,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:14,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:14,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:17,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:17,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:17,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:17,825 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:18,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:19,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:20,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:22,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:22,689 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:22,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:22,694 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:23,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:24,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:25,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:27,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:27,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:27,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:27,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:27,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:28,622 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:29,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:29,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:31,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:33,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:33,064 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:33,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:33,070 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:33,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:34,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:37,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:37,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:37,771 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:37,772 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:37,776 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:38,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:39,161 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:19:40,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:42,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:44,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:44,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:19:46,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:46,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:46,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:46,142 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:46,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:47,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:48,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:50,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:50,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:50,641 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:50,646 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:51,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:52,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:52,651 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:55,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:19:55,248 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:19:55,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:19:55,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:19:55,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:19:56,656 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:19:58,255 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:19:59,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:19:59,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:19:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:03,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:03,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:03,718 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:03,718 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:03,723 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:04,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:06,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:08,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:08,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:08,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:08,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:08,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:09,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:20:09,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:10,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:14,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:14,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:14,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:14,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:14,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:14,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:14,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:15,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:16,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:19,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:20,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:20,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:20,239 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:20,244 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:20,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:22,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:24,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:24,516 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:24,517 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:24,522 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:24,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:25,523 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:26,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:29,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:29,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:29,842 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:29,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:29,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:29,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:30,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:30,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:32,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:34,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:34,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:34,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:34,389 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:34,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:36,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:36,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:39,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:20:40,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:40,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:40,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:40,776 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:41,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:41,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:42,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:44,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:44,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:20:46,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:46,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:46,126 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:46,131 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:46,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:47,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:48,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:52,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:52,521 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:52,522 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:52,522 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:52,528 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:52,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:54,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:56,822 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:20:56,823 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:20:56,823 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:20:56,829 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:20:57,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:20:57,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:20:58,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:20:59,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:20:59,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:20:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:03,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:06,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:06,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:06,750 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:06,755 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:07,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:08,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:08,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:09,164 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:21:11,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:11,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:11,169 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:11,174 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:11,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:12,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:14,175 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:14,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:14,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:15,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:15,473 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:15,473 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:15,478 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:15,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:18,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:19,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:20,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:20,301 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:20,302 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:20,307 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:20,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:22,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:25,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:26,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:26,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:26,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:26,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:27,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:28,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:29,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:29,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:30,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:31,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:31,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:31,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:31,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:31,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:32,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:35,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:35,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:35,210 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:35,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:35,768 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:36,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:36,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:39,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:21:41,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:41,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:41,991 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:41,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:41,996 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:42,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:44,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:44,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:21:44,778 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:46,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:46,688 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:46,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:46,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:46,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:47,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:48,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:51,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:51,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:51,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:51,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:51,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:52,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:54,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:56,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:21:56,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:21:56,023 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:21:56,028 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:21:56,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:21:58,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:21:58,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:21:59,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:21:59,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:21:59,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:00,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:00,679 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:00,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:00,684 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:00,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:02,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:03,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:05,405 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:05,406 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:05,406 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:05,411 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:05,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:06,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:09,168 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:22:09,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:11,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:11,340 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:11,340 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:11,346 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:11,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:12,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:14,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:14,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:14,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:14,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:16,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:16,139 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:16,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:16,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:16,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:18,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:20,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:20,176 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:20,177 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:20,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:20,182 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:20,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:22,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:25,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:26,823 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:26,824 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:26,824 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:26,829 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:27,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:28,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:29,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:29,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:29,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:31,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:31,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:31,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:31,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:31,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:32,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:36,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:36,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:36,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:36,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:36,760 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:36,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:38,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:39,170 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:22:41,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:41,471 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:41,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:41,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:41,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:41,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:42,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:44,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:44,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:44,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:22:45,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:45,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:45,527 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:45,532 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:45,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:46,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:48,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:50,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:50,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:50,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:50,472 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:50,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:52,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:52,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:54,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:22:54,743 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:22:54,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:22:54,748 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:22:54,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:22:56,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:22:57,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:22:59,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:22:59,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:22:59,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:01,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:01,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:01,444 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:01,449 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:01,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:02,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:03,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:06,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:06,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:06,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:06,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:06,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:08,509 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:08,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:09,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:23:10,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:10,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:10,895 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:10,900 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:11,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:12,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:13,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:14,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:14,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:14,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:15,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:15,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:15,943 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:15,948 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:16,876 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:18,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:18,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:23,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:23,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:23,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:23,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:23,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:24,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:24,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:28,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:28,005 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:28,005 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:28,010 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:28,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:29,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:29,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:29,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:29,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:30,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:33,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:33,700 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:33,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:33,705 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:33,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:34,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:36,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:38,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:38,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:38,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:38,717 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:38,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:39,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:23:40,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:40,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:42,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:42,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:42,978 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:42,982 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:43,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:44,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:44,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:44,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:23:44,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:45,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:47,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:47,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:47,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:47,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:47,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:48,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:51,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:51,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:51,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:51,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:51,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:51,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:54,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:56,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:23:56,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:23:56,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:23:56,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:23:56,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:23:56,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:23:58,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:23:59,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:23:59,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:23:59,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:01,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:04,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:04,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:04,884 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:04,889 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:04,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:06,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:06,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:09,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:24:11,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:11,219 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:11,219 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:11,224 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:11,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:12,225 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:12,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:14,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:14,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:14,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:15,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:15,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:15,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:15,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:15,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:16,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:18,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:19,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:19,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:19,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:19,798 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:19,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:22,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:23,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:23,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:23,893 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:23,893 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:23,898 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:23,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:26,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:28,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:29,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:29,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:29,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:29,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:29,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:29,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:32,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:33,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:33,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:33,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:33,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:33,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:34,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:36,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:39,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:24:39,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:39,717 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:39,718 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:39,723 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:39,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:40,723 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:42,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:44,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:44,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:44,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:45,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:45,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:45,610 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:45,615 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:45,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:46,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:46,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:49,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:49,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:49,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:49,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:49,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:51,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:52,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:54,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:54,261 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:54,262 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:54,267 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:54,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:24:56,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:24:57,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:24:59,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:24:59,007 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:24:59,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:24:59,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:24:59,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:24:59,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:24:59,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:24:59,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:00,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:02,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:05,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:05,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:05,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:05,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:05,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:06,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:08,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:09,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:25:10,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:10,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:10,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:10,561 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:10,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:12,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:13,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:14,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:16,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:16,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:16,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:16,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:17,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:19,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:19,329 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:20,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:20,945 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:20,945 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:20,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:21,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:23,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:24,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:25,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:25,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:25,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:25,776 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:26,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:29,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:29,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:30,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:30,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:30,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:30,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:30,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:31,015 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:33,018 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:35,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:36,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:36,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:36,275 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:36,280 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:37,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:39,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:39,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:25:40,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:40,992 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:40,993 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:40,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:40,998 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:41,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:43,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:44,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:44,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:44,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:25:45,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:45,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:45,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:45,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:46,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:46,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:47,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:51,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:51,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:51,836 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:51,836 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:51,841 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:52,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:55,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:56,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:25:56,313 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:25:56,313 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:25:56,318 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:25:57,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:25:57,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:25:59,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:25:59,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:25:59,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:25:59,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:01,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:01,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:01,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:01,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:02,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:02,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:03,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:05,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:05,994 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:05,994 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:05,999 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:06,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:07,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:09,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:09,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:26:10,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:10,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:10,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:10,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:11,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:13,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:13,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:14,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:17,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:17,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:17,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:17,444 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:18,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:18,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:19,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:22,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:22,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:22,384 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:22,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:23,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:24,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:25,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:26,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:26,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:26,828 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:26,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:27,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:29,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:29,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:29,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:32,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:32,552 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:32,552 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:32,557 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:33,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:34,557 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:35,100 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:38,822 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:38,823 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:38,823 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:38,828 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:39,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:39,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:26:40,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:41,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:44,254 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:44,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:44,255 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:44,260 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:44,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:44,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:44,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:26:45,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:45,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:47,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:50,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:51,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:51,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:51,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:51,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:52,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:53,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:55,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:26:56,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:26:56,489 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:26:56,490 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:26:56,495 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:26:57,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:26:59,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:26:59,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:26:59,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:26:59,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:01,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:01,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:01,839 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:01,839 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:01,844 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:02,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:05,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:06,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:06,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:06,143 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:06,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:07,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:07,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:09,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:09,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:27:12,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:13,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:13,741 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:13,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:13,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:14,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:14,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:14,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:15,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:17,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:20,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:20,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:20,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:20,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:21,142 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:23,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:23,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:25,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:25,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:25,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:25,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:25,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:27,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:29,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:29,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:29,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:29,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:29,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:29,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:29,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:30,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:31,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:34,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:34,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:34,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:34,859 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:34,864 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:35,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:37,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:39,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:27:40,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:41,207 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:41,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:41,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:41,213 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:42,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:43,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:44,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:44,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:44,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:27:45,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:45,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:45,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:45,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:45,488 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:46,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:47,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:50,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:50,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:50,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:50,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:50,715 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:51,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:53,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:55,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:27:57,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:27:57,200 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:27:57,200 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:27:57,205 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:27:58,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:27:59,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:27:59,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:27:59,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:27:59,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:01,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:02,242 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:02,242 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:02,243 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:02,248 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:03,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:05,206 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:07,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:07,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:07,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:07,342 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:07,347 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:08,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:09,193 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:28:09,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:13,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:14,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:14,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:14,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:14,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:14,742 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:14,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:14,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:15,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:17,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:18,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:19,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:19,646 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:19,646 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:19,651 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:20,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:21,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:24,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:24,001 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:24,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:24,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:24,006 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:24,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:27,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:28,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:28,902 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:28,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:28,908 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:29,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:29,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:29,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:29,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:31,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:34,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:34,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:34,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:34,422 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:35,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:35,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:37,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:38,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:38,650 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:38,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:38,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:39,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:28:39,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:41,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:41,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:43,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:43,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:43,269 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:43,274 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:44,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:44,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:44,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:44,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:28:45,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:46,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:47,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:47,496 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:47,496 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:47,501 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:48,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:49,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:52,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:52,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:52,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:52,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:52,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:53,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:55,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:57,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:28:57,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:28:57,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:28:57,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:28:57,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:28:58,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:28:59,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:28:59,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:28:59,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:28:59,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:02,329 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:02,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:02,330 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:02,335 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:03,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:03,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:05,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:06,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:06,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:06,600 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:06,605 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:07,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:08,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:09,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:29:09,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:11,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:11,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:11,479 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:11,484 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:12,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:13,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:14,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:14,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:17,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:17,533 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:17,534 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:17,539 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:18,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:19,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:19,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:21,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:21,641 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:21,641 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:21,646 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:22,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:23,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:24,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:29,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:29,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:29,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:30,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:30,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:30,343 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:30,348 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:31,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:33,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:34,569 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:34,570 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:34,570 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:34,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:35,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:35,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:37,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:39,200 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:29:39,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:39,403 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:39,403 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:39,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:40,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:41,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:41,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:43,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:43,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:43,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:43,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:44,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:44,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:44,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:44,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:29:45,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:46,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:47,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:47,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:47,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:47,993 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:48,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:49,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:51,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:53,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:53,936 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:53,936 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:53,941 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:54,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:55,325 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:29:57,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:29:59,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:29:59,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:29:59,028 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:29:59,033 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:29:59,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:29:59,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:29:59,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:29:59,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:01,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:03,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:03,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:03,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:03,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:03,550 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:04,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:05,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:07,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:07,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:07,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:07,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:08,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:08,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:09,205 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:30:09,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:12,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:12,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:12,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:12,661 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:13,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:14,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:14,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:14,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:15,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:17,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:17,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:17,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:17,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:18,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:19,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:19,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:24,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:26,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:26,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:26,006 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:26,011 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:26,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:27,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:29,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:29,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:29,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:31,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:31,310 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:31,310 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:31,316 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:31,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:33,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:35,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:36,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:36,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:36,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:36,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:37,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:39,204 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:30:39,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:41,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:41,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:41,628 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:41,629 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:41,634 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:42,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:43,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:44,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:44,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:44,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:46,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:47,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:47,260 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:47,261 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:47,265 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:47,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:49,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:52,266 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:54,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:54,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:54,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:54,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:30:55,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:30:57,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:30:57,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:30:59,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:30:59,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:30:59,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:30:59,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:30:59,958 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:30:59,958 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:30:59,963 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:00,396 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:01,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:02,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:04,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:04,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:04,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:04,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:05,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:07,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:08,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:09,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:31:09,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:09,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:09,324 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:09,329 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:09,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:11,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:14,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:14,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:15,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:15,999 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:15,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:16,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:16,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:17,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:20,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:25,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:25,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:25,044 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:25,044 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:25,050 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:25,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:27,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:28,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:28,792 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:28,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:28,798 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:29,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:29,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:30,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:31,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:34,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:34,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:34,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:34,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:34,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:36,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:37,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:38,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:38,899 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:38,900 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:38,905 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:39,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:31:39,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:41,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:41,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:44,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:44,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:44,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:46,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:48,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:48,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:48,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:48,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:48,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:49,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:52,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:52,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:52,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:52,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:52,687 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:53,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:55,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:31:57,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:31:58,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:31:58,005 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:31:58,005 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:31:58,010 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:31:58,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:31:59,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:31:59,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:31:59,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:31:59,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:02,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:02,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:02,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:02,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:03,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:03,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:05,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:07,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:07,391 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:07,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:07,396 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:07,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:09,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:32:09,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:09,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:14,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:14,027 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:14,027 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:14,032 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:14,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:14,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:14,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:15,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:19,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:19,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:19,180 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:19,185 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:19,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:20,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:21,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:25,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:25,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:25,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:25,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:25,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:25,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:27,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:29,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:30,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:30,064 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:30,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:30,069 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:30,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:31,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:31,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:35,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:35,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:35,006 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:35,011 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:35,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:37,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:37,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:39,212 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:32:40,142 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:40,143 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:40,143 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:40,148 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:40,508 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:41,511 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:42,149 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:44,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:44,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:44,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:44,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:44,650 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:44,650 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:44,655 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:45,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:47,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:47,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:52,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:52,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:52,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:52,125 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:52,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:53,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:53,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:32:57,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:32:57,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:32:57,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:32:57,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:32:57,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:32:59,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:32:59,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:32:59,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:32:59,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:32:59,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:01,759 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:01,760 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:01,760 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:01,766 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:02,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:03,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:04,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:09,215 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:33:09,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:09,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:09,338 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:09,343 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:09,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:10,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:11,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:14,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:14,301 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:14,301 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:14,306 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:14,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:14,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:14,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:15,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:17,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:20,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:20,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:20,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:20,268 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:20,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:21,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:21,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:25,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:25,212 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:25,212 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:25,217 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:25,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:27,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:27,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:29,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:30,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:30,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:30,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:30,602 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:31,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:32,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:33,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:37,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:38,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:38,995 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:38,995 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:39,000 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:39,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:33:39,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:41,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:43,051 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:43,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:43,052 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:43,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:43,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:43,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:44,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:44,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:44,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:33:45,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:48,569 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:48,570 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:48,570 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:48,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:48,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:48,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:51,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:52,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:33:52,623 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:33:52,623 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:33:52,629 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:33:53,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:33:53,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:55,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:33:58,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:33:59,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:33:59,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:33:59,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:00,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:01,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:01,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:01,006 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:01,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:03,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:04,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:06,821 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:06,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:06,822 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:06,827 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:07,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:09,219 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:34:09,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:09,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:13,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:13,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:13,258 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:13,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:13,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:14,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:14,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:14,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:14,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:15,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:19,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:20,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:20,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:20,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:20,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:20,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:21,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:24,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:24,449 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:24,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:24,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:24,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:25,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:27,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:29,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:29,350 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:29,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:29,355 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:29,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:29,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:29,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:30,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:31,646 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:34,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:34,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:34,275 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:34,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:34,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:35,651 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:36,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:39,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:34:39,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:39,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:39,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:39,285 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:39,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:41,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:41,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:44,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:44,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:44,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:44,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:44,672 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:44,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:44,678 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:44,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:46,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:47,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:48,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:48,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:48,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:48,714 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:49,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:51,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:51,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:53,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:53,741 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:53,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:53,746 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:54,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:34:55,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:34:56,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:34:58,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:34:58,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:34:58,780 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:34:58,785 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:34:59,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:34:59,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:34:59,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:34:59,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:01,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:02,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:03,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:03,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:03,030 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:03,036 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:03,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:05,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:07,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:07,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:07,479 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:07,483 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:07,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:08,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:09,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:35:09,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:12,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:12,761 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:12,761 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:12,766 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:13,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:13,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:14,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:14,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:14,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:15,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:19,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:19,402 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:19,403 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:19,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:19,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:19,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:21,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:24,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:28,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:28,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:28,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:28,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:28,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:29,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:29,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:29,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:34,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:37,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:37,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:37,475 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:37,480 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:37,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:39,225 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:35:39,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:40,227 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:42,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:42,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:42,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:42,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:42,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:43,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:44,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:44,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:44,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:46,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:46,387 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:46,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:46,392 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:46,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:47,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:51,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:51,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:51,974 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:51,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:51,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:52,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:35:53,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:35:56,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:35:59,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:35:59,057 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:35:59,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:35:59,062 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:35:59,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:35:59,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:35:59,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:35:59,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:01,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:02,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:03,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:03,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:03,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:03,370 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:03,763 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:05,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:08,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:08,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:08,475 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:08,475 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:08,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:08,768 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:09,227 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:36:11,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:13,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:13,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:13,859 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:13,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:13,867 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:14,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:14,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:14,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:14,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:15,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:19,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:19,449 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:19,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:19,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:19,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:19,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:21,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:24,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:27,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:27,794 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:27,794 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:27,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:28,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:29,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:29,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:29,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:29,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:29,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:32,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:32,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:32,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:32,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:32,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:33,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:35,298 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:36,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:36,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:36,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:36,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:36,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:39,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:36:39,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:41,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:42,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:42,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:42,565 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:42,570 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:42,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:44,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:44,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:44,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:36:45,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:46,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:47,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:47,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:47,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:47,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:47,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:49,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:52,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:55,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:36:55,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:36:55,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:36:55,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:36:55,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:36:56,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:36:57,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:36:59,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:36:59,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:36:59,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:00,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:00,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:00,289 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:00,294 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:00,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:00,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:03,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:04,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:04,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:04,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:04,774 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:04,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:06,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:08,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:09,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:37:13,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:13,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:13,013 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:13,018 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:13,865 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:14,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:14,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:14,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:14,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:14,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:17,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:17,497 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:17,497 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:17,502 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:17,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:18,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:19,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:22,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:22,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:22,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:22,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:22,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:22,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:25,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:27,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:27,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:27,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:27,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:27,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:28,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:29,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:29,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:29,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:30,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:32,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:32,213 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:32,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:32,219 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:32,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:32,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:36,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:37,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:37,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:37,194 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:37,199 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:37,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:38,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:39,234 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:37:41,235 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:43,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:43,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:43,527 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:43,532 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:43,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:44,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:44,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:44,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:37:44,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:46,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:48,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:48,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:48,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:48,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:48,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:50,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:51,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:53,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:53,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:53,874 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:53,879 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:53,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:54,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:56,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:37:58,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:37:58,291 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:37:58,291 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:37:58,296 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:37:58,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:37:58,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:37:59,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:37:59,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:37:59,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:02,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:02,553 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:02,553 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:02,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:02,558 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:02,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:02,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:06,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:06,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:06,984 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:06,989 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:07,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:07,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:08,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:09,239 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:38:12,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:12,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:12,888 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:12,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:12,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:13,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:14,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:14,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:14,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:14,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:17,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:17,954 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:17,954 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:17,959 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:17,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:18,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:18,966 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:22,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:22,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:22,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:22,618 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:22,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:24,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:24,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:28,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:28,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:28,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:28,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:28,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:29,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:29,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:29,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:30,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:34,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:35,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:35,170 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:35,170 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:35,175 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:35,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:36,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:39,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:38:39,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:39,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:39,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:39,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:39,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:40,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:40,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:44,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:44,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:44,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:38:45,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:45,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:45,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:45,602 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:45,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:46,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:46,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:51,333 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:51,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:51,334 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:51,339 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:51,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:52,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:53,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:57,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:38:57,177 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:38:57,177 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:38:57,182 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:38:58,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:38:58,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:38:59,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:38:59,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:38:59,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:38:59,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:02,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:02,913 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:02,913 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:02,918 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:03,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:03,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:05,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:08,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:08,026 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:08,026 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:08,031 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:09,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:09,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:09,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:09,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:39:13,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:13,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:13,172 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:13,177 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:14,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:14,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:14,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:14,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:14,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:15,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:19,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:21,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:21,512 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:21,512 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:21,517 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:22,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:23,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:25,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:26,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:26,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:26,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:26,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:27,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:29,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:29,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:29,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:29,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:30,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:33,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:33,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:33,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:33,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:34,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:35,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:36,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:37,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:37,601 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:37,601 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:37,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:38,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:39,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:39,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:39:41,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:42,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:42,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:42,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:42,030 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:42,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:43,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:44,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:44,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:44,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:46,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:46,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:46,645 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:46,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:46,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:47,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:47,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:51,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:55,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:55,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:55,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:55,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:39:55,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:39:57,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:39:57,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:39:59,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:39:59,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:39:59,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:39:59,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:39:59,882 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:39:59,882 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:39:59,887 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:00,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:01,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:02,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:04,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:04,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:04,783 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:04,789 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:05,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:07,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:08,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:09,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:40:10,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:10,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:10,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:10,048 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:10,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:11,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:14,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:14,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:14,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:14,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:15,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:15,312 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:15,312 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:15,317 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:16,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:17,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:19,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:19,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:19,851 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:19,851 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:19,856 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:20,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:21,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:24,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:26,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:26,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:26,364 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:26,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:27,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:27,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:29,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:29,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:31,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:31,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:31,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:31,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:32,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:33,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:35,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:35,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:35,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:35,533 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:35,538 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:36,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:37,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:39,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:40:40,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:40,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:40,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:40,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:40,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:41,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:41,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:44,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:44,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:44,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:40:45,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:49,823 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:49,824 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:49,824 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:49,829 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:50,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:51,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:51,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:54,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:54,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:54,485 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:54,490 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:55,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:55,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:57,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:40:58,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:40:58,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:40:58,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:40:58,376 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:40:59,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:40:59,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:40:59,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:40:59,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:40:59,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:02,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:03,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:03,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:03,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:03,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:04,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:05,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:08,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:08,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:08,284 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:08,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:08,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:09,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:09,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:09,248 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:41:12,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:12,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:12,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:12,774 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:13,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:13,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:13,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:14,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:14,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:14,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:17,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:17,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:17,222 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:17,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:18,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:19,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:19,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:24,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:25,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:25,723 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:25,723 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:25,728 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:26,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:27,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:29,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:29,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:29,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:29,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:29,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:30,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:31,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:34,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:35,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:35,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:35,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:35,045 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:35,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:37,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:39,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:41:39,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:39,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:39,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:39,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:40,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:40,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:41,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:43,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:43,945 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:43,946 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:43,951 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:44,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:44,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:44,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:44,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:41:45,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:45,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:48,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:48,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:48,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:48,780 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:49,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:49,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:51,816 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:53,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:53,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:53,180 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:53,185 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:53,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:55,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:57,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:41:58,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:41:58,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:41:58,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:41:58,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:41:59,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:41:59,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:41:59,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:41:59,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:41:59,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:02,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:02,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:02,627 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:02,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:02,632 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:03,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:03,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:07,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:07,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:07,787 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:07,787 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:07,792 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:08,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:09,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:09,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:42:13,255 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:14,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:14,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:14,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:15,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:15,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:15,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:15,373 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:16,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:17,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:18,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:20,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:20,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:20,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:20,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:20,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:21,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:24,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:29,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:29,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:29,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:29,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:29,618 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:29,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:29,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:30,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:31,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:33,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:33,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:33,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:33,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:34,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:34,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:35,280 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:37,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:37,994 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:37,995 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:38,000 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:38,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:39,255 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:42:39,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:40,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:42,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:42,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:42,251 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:42,256 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:42,286 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:43,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:44,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:44,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:42:45,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:47,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:47,580 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:47,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:47,586 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:48,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:49,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:51,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:51,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:51,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:51,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:51,742 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:52,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:53,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:56,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:42:57,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:42:57,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:42:57,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:42:57,473 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:42:58,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:42:59,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:42:59,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:42:59,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:42:59,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:01,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:02,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:02,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:02,273 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:02,278 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:02,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:03,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:07,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:09,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:43:10,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:10,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:10,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:10,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:11,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:11,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:12,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:14,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:14,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:14,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:15,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:15,032 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:15,032 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:15,037 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:15,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:17,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:18,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:21,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:21,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:21,514 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:21,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:22,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:23,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:23,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:26,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:26,702 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:26,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:26,708 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:27,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:27,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:28,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:29,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:29,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:32,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:32,786 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:32,787 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:32,792 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:33,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:33,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:33,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:37,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:37,073 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:37,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:37,078 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:37,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:39,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:39,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:43:39,350 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:44,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:44,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:43:45,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:45,382 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:45,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:45,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:46,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:47,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:49,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:49,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:49,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:49,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:49,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:50,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:51,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:54,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:43:54,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:43:54,297 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:43:54,302 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:43:54,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:43:55,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:43:55,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:43:59,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:43:59,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:43:59,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:00,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:02,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:02,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:02,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:02,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:03,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:03,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:06,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:08,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:08,242 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:08,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:08,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:08,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:09,264 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:44:09,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:12,266 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:13,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:13,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:13,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:13,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:14,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:14,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:14,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:14,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:15,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:17,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:19,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:19,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:19,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:19,371 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:19,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:21,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:23,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:26,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:26,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:26,527 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:26,532 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:27,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:27,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:28,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:29,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:33,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:33,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:33,930 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:33,930 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:33,935 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:34,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:35,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:37,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:37,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:37,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:37,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:38,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:38,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:39,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:44:39,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:44,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:44,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:44:47,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:47,067 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:47,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:47,072 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:47,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:49,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:50,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:51,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:51,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:51,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:51,550 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:52,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:53,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:55,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:44:57,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:44:57,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:44:57,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:44:57,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:44:57,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:44:59,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:44:59,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:44:59,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:44:59,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:00,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:01,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:01,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:01,548 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:01,553 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:02,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:03,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:06,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:06,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:06,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:06,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:06,711 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:07,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:07,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:09,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:45:12,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:12,033 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:12,033 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:12,033 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:12,038 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:12,483 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:13,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:14,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:14,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:14,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:17,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:20,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:20,480 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:20,480 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:20,485 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:20,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:21,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:23,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:28,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:28,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:28,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:28,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:28,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:29,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:29,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:29,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:33,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:37,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:37,702 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:37,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:37,707 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:38,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:39,267 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:45:39,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:39,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:42,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:42,363 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:42,364 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:42,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:42,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:43,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:44,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:44,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:45:46,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:46,661 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:46,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:46,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:47,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:47,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:49,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:52,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:52,688 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:52,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:52,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:53,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:53,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:55,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:45:56,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:45:56,614 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:45:56,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:45:56,619 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:45:57,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:45:57,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:45:59,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:45:59,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:45:59,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:00,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:01,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:01,749 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:01,749 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:01,754 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:02,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:03,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:05,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:05,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:05,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:05,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:05,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:06,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:07,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:09,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:46:10,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:10,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:10,780 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:10,785 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:11,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:11,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:11,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:14,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:14,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:14,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:14,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:14,887 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:14,887 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:14,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:15,585 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:15,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:16,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:19,593 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:19,594 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:19,594 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:19,600 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:19,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:21,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:22,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:24,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:24,565 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:24,565 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:24,571 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:24,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:25,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:28,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:29,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:29,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:29,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:30,266 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:30,267 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:30,267 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:30,272 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:30,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:31,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:34,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:34,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:34,680 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:34,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:34,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:35,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:35,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:39,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:46:40,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:40,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:40,958 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:40,959 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:40,964 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:41,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:41,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:44,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:44,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:44,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:45,663 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:45,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:45,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:45,668 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:45,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:47,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:50,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:50,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:50,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:50,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:50,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:51,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:51,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:54,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:54,950 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:54,951 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:54,956 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:55,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:46:55,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:46:56,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:46:59,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:46:59,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:46:59,361 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:46:59,367 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:46:59,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:46:59,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:46:59,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:46:59,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:01,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:02,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:04,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:04,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:04,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:04,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:04,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:05,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:08,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:09,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:47:12,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:12,447 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:12,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:12,453 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:12,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:13,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:13,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:14,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:14,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:14,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:16,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:16,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:16,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:16,970 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:17,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:17,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:19,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:23,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:23,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:23,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:23,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:23,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:23,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:24,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:29,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:29,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:29,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:29,115 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:29,119 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:29,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:29,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:29,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:29,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:29,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:34,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:36,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:36,993 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:36,993 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:36,999 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:37,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:37,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:39,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:47:40,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:42,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:42,563 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:42,563 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:42,568 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:42,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:43,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:44,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:44,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:44,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:45,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:46,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:46,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:46,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:46,819 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:47,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:47,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:51,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:51,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:51,276 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:51,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:51,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:47:51,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:47:53,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:47:56,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:47:59,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:47:59,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:47:59,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:47:59,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:47:59,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:47:59,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:47:59,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:00,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:01,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:01,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:04,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:04,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:04,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:04,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:04,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:05,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:07,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:09,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:48:10,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:10,902 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:10,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:10,907 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:11,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:11,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:12,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:14,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:14,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:14,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:15,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:15,963 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:15,963 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:15,968 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:16,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:17,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:18,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:21,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:21,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:21,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:21,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:21,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:23,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:23,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:26,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:26,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:26,547 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:26,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:26,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:27,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:28,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:29,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:29,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:32,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:32,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:32,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:32,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:32,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:33,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:34,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:36,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:36,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:36,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:36,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:36,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:37,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:39,280 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:48:39,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:42,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:42,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:42,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:42,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:42,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:43,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:44,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:44,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:44,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:44,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:46,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:46,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:46,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:46,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:47,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:47,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:49,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:54,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:55,286 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:55,287 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:55,287 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:55,293 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:48:55,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:48:55,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:48:59,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:48:59,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:48:59,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:48:59,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:48:59,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:48:59,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:48:59,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:48:59,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:00,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:01,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:04,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:04,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:04,337 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:04,342 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:04,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:05,343 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:05,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:09,282 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:49:09,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:09,578 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:09,578 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:09,583 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:09,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:10,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:11,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:14,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:14,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:14,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:15,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:15,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:15,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:15,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:15,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:16,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:17,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:19,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:19,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:19,910 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:19,915 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:20,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:21,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:21,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:24,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:24,546 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:24,547 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:24,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:24,865 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:25,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:27,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:29,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:29,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:29,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:29,437 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:29,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:29,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:29,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:31,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:32,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:35,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:35,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:35,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:35,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:35,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:35,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:38,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:39,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:49:40,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:40,920 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:40,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:40,925 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:41,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:41,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:43,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:44,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:44,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:44,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:49:47,082 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:47,083 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:47,083 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:47,088 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:47,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:47,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:49,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:54,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:56,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:49:56,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:49:56,186 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:49:56,191 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:49:56,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:49:57,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:49:59,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:49:59,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:49:59,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:49:59,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:01,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:01,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:01,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:01,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:01,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:01,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:05,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:05,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:05,486 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:05,486 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:05,491 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:05,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:07,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:09,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:50:10,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:10,213 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:10,213 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:10,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:10,218 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:10,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:11,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:14,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:14,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:14,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:15,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:17,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:17,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:17,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:17,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:17,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:19,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:21,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:22,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:22,296 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:22,296 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:22,315 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:22,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:23,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:27,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:27,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:27,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:27,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:27,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:27,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:27,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:29,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:29,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:29,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:32,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:33,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:33,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:33,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:33,065 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:33,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:33,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:38,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:38,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:38,703 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:38,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:38,708 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:38,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:39,288 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:50:39,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:43,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:43,246 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:43,246 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:43,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:43,252 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:43,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:43,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:44,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:44,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:44,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:47,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:47,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:47,849 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:47,856 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:47,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:48,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:49,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:53,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:54,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:54,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:54,276 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:54,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:54,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:50:55,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:59,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:50:59,062 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:50:59,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:50:59,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:50:59,068 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:50:59,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:50:59,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:50:59,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:50:59,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:50:59,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:04,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:05,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:05,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:05,335 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:05,340 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:05,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:05,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:09,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:51:10,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:12,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:12,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:12,783 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:12,788 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:12,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:13,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:14,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:14,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:14,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:15,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:18,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:18,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:18,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:18,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:19,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:20,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:21,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:22,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:22,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:22,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:22,446 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:23,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:24,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:26,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:27,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:27,764 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:27,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:27,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:28,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:29,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:29,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:30,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:31,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:33,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:33,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:33,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:33,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:34,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:34,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:37,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:37,461 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:37,462 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:37,462 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:37,467 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:38,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:38,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:39,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:51:42,294 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:43,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:43,666 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:43,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:43,671 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:44,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:44,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:44,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:44,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:51:46,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:47,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:48,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:48,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:48,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:48,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:49,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:50,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:53,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:53,177 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:53,177 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:53,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:53,183 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:54,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:51:54,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:57,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:51:57,639 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:51:57,639 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:51:57,644 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:51:58,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:51:58,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:51:59,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:51:59,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:51:59,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:00,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:03,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:04,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:04,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:04,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:04,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:05,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:06,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:08,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:08,641 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:08,641 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:08,647 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:09,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:09,295 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:52:09,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:10,058 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:14,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:14,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:14,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:14,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:14,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:14,950 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:14,950 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:14,955 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:15,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:16,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:19,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:19,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:19,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:19,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:19,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:20,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:20,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:23,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:23,382 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:23,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:23,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:24,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:24,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:25,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:27,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:27,288 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:27,288 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:27,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:28,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:28,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:29,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:30,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:32,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:32,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:32,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:32,721 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:33,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:34,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:36,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:39,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:52:41,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:41,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:41,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:41,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:41,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:42,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:42,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:44,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:44,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:44,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:52:45,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:45,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:45,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:45,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:46,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:47,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:48,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:50,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:50,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:50,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:50,401 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:51,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:52,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:53,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:55,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:52:55,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:52:55,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:52:55,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:52:56,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:52:56,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:52:59,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:52:59,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:52:59,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:52:59,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:00,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:00,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:00,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:00,006 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:00,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:02,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:04,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:04,545 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:04,546 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:04,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:04,551 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:05,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:06,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:09,299 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:53:10,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:14,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:14,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:14,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:15,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:16,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:16,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:16,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:16,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:16,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:18,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:20,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:20,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:20,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:20,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:21,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:21,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:22,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:26,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:26,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:26,604 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:26,604 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:26,610 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:27,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:28,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:29,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:31,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:32,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:32,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:32,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:32,776 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:33,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:34,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:37,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:38,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:38,307 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:38,308 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:38,313 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:39,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:39,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:53:40,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:43,306 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:44,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:44,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:44,646 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:44,651 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:44,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:44,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:53:44,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:45,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:46,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:48,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:50,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:50,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:50,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:50,326 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:51,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:52,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:54,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:54,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:53:54,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:53:54,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:53:54,805 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:53:55,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:53:56,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:53:59,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:53:59,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:53:59,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:53:59,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:00,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:00,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:00,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:01,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:02,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:04,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:05,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:05,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:05,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:05,400 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:06,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:06,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:09,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:54:10,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:10,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:10,715 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:10,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:10,720 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:11,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:12,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:14,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:15,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:15,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:15,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:15,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:15,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:16,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:18,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:21,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:21,761 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:21,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:21,762 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:21,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:22,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:24,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:26,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:29,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:29,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:29,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:30,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:30,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:30,251 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:30,256 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:30,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:32,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:32,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:34,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:34,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:34,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:34,929 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:35,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:36,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:37,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:39,305 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:54:43,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:44,242 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:44,243 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:44,243 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:44,248 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:44,372 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:44,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:44,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:44,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:54:46,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:48,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:49,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:49,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:49,544 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:49,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:50,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:50,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:54,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:54,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:54,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:54,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:54,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:55,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:54:56,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:54:59,518 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:54:59,519 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:54:59,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:54:59,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:54:59,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:54:59,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:54:59,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:54:59,658 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:00,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:00,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:04,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:04,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:04,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:04,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:04,993 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:05,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:06,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:09,307 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:55:10,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:13,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:13,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:13,361 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:13,366 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:13,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:14,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:14,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:15,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:18,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:18,301 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:18,301 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:18,306 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:18,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:20,413 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:21,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:26,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:26,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:26,726 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:26,726 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:26,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:27,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:28,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:29,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:29,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:29,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:31,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:31,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:31,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:31,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:31,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:31,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:32,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:36,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:36,563 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:36,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:36,564 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:36,570 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:37,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:38,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:39,310 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:55:42,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:43,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:43,141 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:43,141 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:43,146 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:43,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:44,436 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:44,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:44,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:44,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:55:47,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:47,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:47,958 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:47,958 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:47,963 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:48,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:50,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:53,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:53,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:53,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:53,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:53,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:54,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:56,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:55:58,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:55:58,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:55:58,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:55:58,242 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:55:58,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:55:58,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:55:59,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:55:59,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:55:59,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:00,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:02,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:03,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:03,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:03,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:03,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:04,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:04,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:07,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:07,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:07,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:07,371 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:07,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:08,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:09,315 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:56:09,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:12,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:12,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:12,879 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:12,884 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:13,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:14,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:14,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:18,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:18,826 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:18,826 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:18,831 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:19,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:19,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:20,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:24,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:24,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:24,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:24,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:24,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:25,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:26,582 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:28,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:28,606 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:28,606 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:28,612 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:29,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:29,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:29,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:29,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:30,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:30,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:34,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:34,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:34,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:34,393 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:34,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:36,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:36,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:38,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:38,626 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:38,626 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:38,631 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:39,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:56:39,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:40,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:42,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:43,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:43,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:43,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:43,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:44,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:44,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:44,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:44,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:44,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:56:47,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:48,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:48,104 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:48,105 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:48,110 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:48,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:50,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:53,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:53,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:53,211 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:53,212 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:53,217 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:53,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:54,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:57,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:56:57,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:56:57,315 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:56:57,320 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:56:57,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:56:58,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:56:58,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:56:59,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:56:59,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:56:59,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:01,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:01,500 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:01,501 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:01,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:01,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:02,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:03,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:05,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:05,797 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:05,797 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:05,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:06,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:06,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:08,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:09,330 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:57:10,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:10,997 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:10,997 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:11,002 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:11,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:12,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:14,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:14,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:14,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:14,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:15,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:15,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:15,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:15,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:15,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:16,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:19,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:19,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:19,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:19,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:19,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:20,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:20,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:24,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:24,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:24,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:24,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:24,679 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:24,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:26,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:28,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:28,764 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:28,764 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:28,769 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:29,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:29,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:29,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:29,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:29,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:30,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:33,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:33,549 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:33,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:33,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:33,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:34,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:35,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:39,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:57:40,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:40,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:40,518 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:40,523 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:40,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:41,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:42,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:44,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:44,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:44,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:44,907 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:44,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:57:45,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:46,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:47,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:49,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:49,158 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:49,158 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:49,164 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:49,730 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:50,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:52,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:54,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:54,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:54,129 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:54,134 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:54,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:56,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:57:58,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:57:59,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:57:59,310 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:57:59,310 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:57:59,315 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:57:59,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:57:59,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:57:59,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:57:59,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:00,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:03,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:03,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:03,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:03,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:03,622 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:03,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:04,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:07,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:07,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:07,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:07,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:07,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:08,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:08,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:09,323 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:58:12,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:12,657 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:12,658 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:12,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:12,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:14,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:14,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:14,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:14,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:17,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:17,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:17,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:17,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:17,763 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:18,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:20,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:25,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:25,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:25,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:25,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:25,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:26,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:26,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:29,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:29,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:29,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:30,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:30,197 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:30,197 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:30,202 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:30,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:32,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:32,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:35,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:35,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:35,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:35,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:35,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:36,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:38,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:39,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:58:41,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:41,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:41,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:41,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:41,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:42,790 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:43,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:44,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:44,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:58:46,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:46,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:46,370 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:46,376 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:46,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:48,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:48,796 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:51,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:51,161 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:51,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:51,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:51,836 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:52,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:54,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:55,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:58:55,772 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:58:55,772 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:58:55,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:58:55,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:58:56,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:58:59,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:58:59,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:58:59,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:58:59,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:02,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:02,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:02,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:02,063 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:02,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:04,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:05,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:07,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:07,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:07,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:07,915 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:08,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:08,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:09,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:59:10,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:12,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:12,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:12,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:12,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:12,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:14,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:14,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:16,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:18,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:18,270 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:18,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:18,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:18,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:20,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:21,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:26,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:26,523 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:26,524 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:26,524 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:26,530 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:26,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:28,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:29,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:29,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:29,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:31,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:31,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:31,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:31,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:31,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:31,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:32,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:35,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:35,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:35,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:35,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:35,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:36,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:36,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:39,330 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 13:59:40,442 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:40,443 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:40,443 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:40,448 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:40,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:42,449 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:42,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:44,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 13:59:45,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:45,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:45,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:45,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:45,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:46,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:47,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:50,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:50,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:50,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:50,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:50,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:52,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:52,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:55,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 13:59:55,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 13:59:55,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 13:59:55,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 13:59:55,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 13:59:56,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 13:59:58,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 13:59:59,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 13:59:59,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 13:59:59,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:00,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:00,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:00,548 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:00,553 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:00,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:02,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:03,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:06,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:06,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:06,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:06,445 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:06,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:08,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:09,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:00:09,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:11,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:11,154 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:11,154 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:11,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:11,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:12,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:14,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:15,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:15,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:15,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:15,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:15,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:16,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:19,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:19,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:19,470 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:19,475 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:19,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:20,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:20,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:23,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:23,732 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:23,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:23,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:23,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:24,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:25,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:28,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:28,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:28,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:28,973 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:29,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:29,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:29,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:29,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:30,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:31,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:33,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:33,694 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:33,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:33,699 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:33,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:34,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:36,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:37,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:37,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:37,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:37,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:37,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:38,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:39,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:00:42,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:43,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:43,419 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:43,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:43,426 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:43,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:44,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:00:44,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:48,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:50,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:50,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:50,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:50,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:50,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:50,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:54,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:55,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:00:55,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:00:55,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:00:55,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:00:55,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:00:56,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:00:59,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:00:59,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:00:59,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:00:59,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:01,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:01,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:01,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:01,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:01,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:02,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:05,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:07,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:07,063 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:07,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:07,068 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:07,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:08,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:09,333 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:01:11,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:12,595 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:12,596 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:12,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:12,602 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:12,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:14,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:14,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:14,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:14,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:16,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:16,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:16,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:16,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:16,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:16,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:19,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:21,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:21,631 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:21,631 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:21,636 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:22,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:22,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:23,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:27,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:29,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:29,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:29,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:31,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:31,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:31,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:31,041 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:32,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:33,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:33,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:37,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:37,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:37,165 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:37,170 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:38,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:38,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:39,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:39,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:01:42,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:42,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:42,514 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:42,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:43,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:43,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:44,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:44,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:01:45,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:47,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:47,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:47,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:47,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:48,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:49,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:49,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:53,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:53,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:53,169 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:53,175 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:54,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:55,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:55,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:01:58,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:01:58,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:01:58,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:01:58,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:01:59,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:01:59,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:01:59,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:01:59,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:01:59,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:01,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:04,157 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:04,158 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:04,158 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:04,163 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:05,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:05,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:06,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:09,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:02:10,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:10,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:10,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:10,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:11,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:11,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:12,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:14,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:14,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:14,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:15,175 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:15,191 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:15,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:15,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:16,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:17,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:17,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:21,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:21,146 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:21,146 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:21,151 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:22,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:23,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:23,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:28,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:29,559 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:29,559 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:29,560 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:29,565 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:29,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:29,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:29,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:30,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:31,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:33,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:33,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:33,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:33,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:33,916 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:34,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:35,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:38,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:39,339 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:02:39,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:39,800 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:39,800 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:39,806 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:40,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:41,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:44,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:44,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:02:47,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:47,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:47,239 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:47,245 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:48,100 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:49,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:50,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:53,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:53,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:53,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:53,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:54,107 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:02:55,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:02:55,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:02:59,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:02:59,552 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:02:59,552 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:02:59,558 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:02:59,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:02:59,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:02:59,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:00,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:01,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:01,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:05,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:05,676 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:05,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:05,681 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:06,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:06,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:07,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:09,344 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:03:09,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:09,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:09,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:09,739 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:10,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:11,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:11,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:14,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:14,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:14,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:14,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:14,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:14,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:14,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:15,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:15,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:17,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:19,258 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:19,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:19,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:19,265 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:20,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:21,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:22,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:23,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:23,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:23,757 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:23,762 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:24,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:25,142 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:27,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:29,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:29,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:29,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:29,428 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:29,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:29,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:29,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:30,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:31,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:33,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:34,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:34,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:34,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:34,805 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:35,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:37,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:38,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:39,174 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:39,175 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:39,175 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:39,181 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:39,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:03:40,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:41,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:44,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:44,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:03:46,790 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:46,791 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:46,791 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:46,796 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:47,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:49,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:49,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:54,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:55,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:55,116 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:55,116 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:55,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:55,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:03:57,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:03:59,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:03:59,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:03:59,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:03:59,206 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:03:59,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:03:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:03:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:03:59,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:00,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:01,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:03,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:03,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:03,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:03,445 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:04,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:05,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:05,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:08,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:08,433 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:08,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:08,439 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:09,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:09,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:04:11,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:11,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:14,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:14,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:14,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:14,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:14,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:14,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:14,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:15,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:15,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:17,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:18,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:18,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:18,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:18,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:19,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:21,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:22,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:24,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:24,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:24,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:24,429 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:25,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:25,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:28,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:29,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:29,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:29,780 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:29,787 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:29,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:29,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:29,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:30,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:31,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:34,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:34,586 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:34,587 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:34,587 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:34,592 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:35,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:37,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:39,047 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:39,048 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:39,048 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:39,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:39,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:39,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:04:39,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:41,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:44,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:44,123 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:44,123 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:44,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:44,232 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:44,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:44,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:44,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:44,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:04:45,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:49,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:49,680 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:49,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:49,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:50,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:50,724 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:51,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:54,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:04:54,212 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:04:54,213 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:04:54,219 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:04:54,243 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:04:55,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:04:56,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:04:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:04:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:04:59,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:00,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:00,409 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:00,409 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:00,415 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:01,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:01,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:01,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:04,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:04,693 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:04,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:04,698 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:05,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:06,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:07,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:09,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:09,335 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:09,335 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:09,340 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:09,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:05:10,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:11,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:12,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:14,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:14,847 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:14,847 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:14,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:14,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:14,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:14,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:15,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:17,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:18,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:19,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:19,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:19,470 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:19,475 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:20,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:21,286 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:23,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:24,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:24,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:24,169 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:24,174 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:24,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:25,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:29,175 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:29,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:29,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:29,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:30,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:30,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:30,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:30,927 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:31,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:33,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:34,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:35,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:35,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:35,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:35,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:36,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:37,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:39,333 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:39,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:39,334 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:39,339 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:39,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:05:40,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:40,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:41,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:44,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:44,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:44,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:44,148 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:44,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:44,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:44,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:44,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:05:45,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:46,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:50,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:50,620 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:50,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:50,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:51,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:51,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:53,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:54,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:54,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:54,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:54,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:55,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:05:56,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:05:57,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:05:59,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:05:59,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:05:59,864 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:05:59,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:05:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:05:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:05:59,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:00,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:01,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:02,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:04,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:04,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:04,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:04,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:04,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:05,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:08,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:08,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:08,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:08,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:08,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:09,355 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:06:09,452 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:11,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:13,357 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:14,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:14,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:14,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:15,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:15,536 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:15,536 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:15,542 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:16,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:17,461 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:18,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:19,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:19,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:19,987 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:19,992 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:20,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:21,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:23,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:26,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:26,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:26,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:26,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:26,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:27,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:29,299 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:29,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:29,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:29,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:31,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:31,200 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:31,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:31,206 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:31,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:33,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:35,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:35,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:35,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:35,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:35,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:36,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:37,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:39,357 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:06:40,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:40,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:40,129 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:40,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:40,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:41,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:41,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:44,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:44,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:06:46,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:48,242 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:48,242 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:48,243 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:48,248 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:48,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:49,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:51,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:52,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:52,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:52,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:52,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:52,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:53,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:06:57,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:06:59,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:06:59,506 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:06:59,506 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:06:59,512 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:06:59,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:06:59,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:06:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:06:59,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:01,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:02,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:04,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:04,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:04,222 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:04,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:04,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:05,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:07,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:09,359 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:07:09,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:09,630 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:09,630 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:09,636 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:10,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:11,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:12,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:14,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:14,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:14,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:16,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:16,363 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:16,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:16,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:16,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:17,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:18,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:20,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:20,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:20,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:20,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:21,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:23,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:23,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:25,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:25,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:25,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:25,917 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:26,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:27,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:28,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:29,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:29,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:29,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:30,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:30,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:30,338 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:30,343 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:30,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:31,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:34,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:38,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:38,743 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:38,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:38,748 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:39,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:07:39,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:40,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:41,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:44,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:07:45,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:45,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:45,231 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:45,237 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:45,575 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:46,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:47,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:50,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:50,930 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:50,930 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:50,935 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:51,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:51,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:53,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:56,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:07:57,465 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:07:57,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:07:57,466 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:07:57,472 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:07:57,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:07:59,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:07:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:07:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:07:59,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:02,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:02,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:02,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:02,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:02,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:03,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:05,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:07,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:07,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:07,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:07,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:07,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:07,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:09,363 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:08:09,601 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:12,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:12,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:12,209 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:12,216 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:12,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:13,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:13,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:14,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:14,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:14,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:17,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:17,363 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:17,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:17,368 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:17,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:18,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:19,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:22,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:22,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:22,404 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:22,409 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:22,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:23,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:23,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:27,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:27,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:27,126 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:27,131 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:27,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:29,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:29,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:29,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:29,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:29,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:34,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:35,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:35,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:35,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:35,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:36,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:37,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:39,368 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:08:39,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:40,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:40,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:40,511 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:40,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:40,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:41,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:44,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:44,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:08:46,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:46,583 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:46,583 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:46,588 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:46,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:47,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:49,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:51,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:51,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:51,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:51,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:51,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:53,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:55,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:08:55,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:08:55,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:08:55,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:08:55,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:08:56,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:08:57,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:08:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:08:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:08:59,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:00,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:01,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:01,800 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:01,801 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:01,806 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:02,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:03,663 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:05,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:08,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:08,067 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:08,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:08,072 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:08,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:09,368 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:09:09,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:11,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:14,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:14,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:14,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:15,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:15,116 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:15,116 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:15,121 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:15,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:17,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:17,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:20,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:20,935 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:20,936 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:20,941 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:21,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:22,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:23,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:26,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:26,177 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:26,177 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:26,182 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:26,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:27,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:28,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:29,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:29,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:29,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:31,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:31,456 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:31,457 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:31,462 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:31,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:33,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:33,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:35,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:35,890 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:35,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:35,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:36,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:37,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:38,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:39,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:09:41,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:41,855 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:41,855 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:41,860 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:42,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:43,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:44,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:44,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:44,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:09:47,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:47,583 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:47,583 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:47,589 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:47,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:49,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:50,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:52,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:52,373 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:52,373 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:52,378 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:52,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:53,734 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:56,379 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:09:56,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:09:56,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:09:56,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:09:56,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:09:57,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:09:59,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:09:59,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:09:59,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:09:59,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:02,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:02,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:02,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:02,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:02,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:03,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:05,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:07,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:09,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:09,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:09,341 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:09,346 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:09,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:10:09,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:11,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:13,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:14,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:14,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:14,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:14,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:14,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:14,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:14,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:14,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:15,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:18,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:18,809 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:18,809 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:18,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:18,814 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:19,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:21,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:22,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:22,906 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:22,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:22,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:23,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:23,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:25,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:27,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:27,833 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:27,833 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:27,838 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:28,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:29,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:29,839 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:29,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:29,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:29,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:32,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:32,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:32,129 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:32,134 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:32,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:33,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:35,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:36,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:36,809 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:36,809 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:36,814 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:36,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:39,375 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:10:39,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:40,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:41,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:41,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:41,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:41,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:41,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:43,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:44,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:44,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:44,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:10:45,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:45,856 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:45,856 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:45,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:45,861 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:46,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:47,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:50,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:50,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:50,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:50,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:50,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:51,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:51,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:55,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:10:55,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:10:55,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:10:55,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:10:55,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:10:56,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:10:57,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:10:59,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:10:59,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:10:59,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:01,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:01,845 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:01,845 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:01,845 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:01,850 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:01,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:02,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:06,311 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:06,312 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:06,312 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:06,317 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:06,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:06,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:07,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:09,380 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:11:12,033 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:12,034 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:12,034 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:12,040 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:12,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:12,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:13,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:14,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:14,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:14,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:16,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:16,540 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:16,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:16,545 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:16,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:18,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:18,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:23,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:25,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:25,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:25,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:25,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:25,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:26,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:29,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:29,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:29,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:29,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:30,826 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:30,826 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:30,827 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:30,831 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:31,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:33,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:34,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:35,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:35,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:35,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:35,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:36,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:37,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:39,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:11:40,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:40,594 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:40,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:40,596 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:40,601 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:41,018 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:43,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:44,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:44,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:44,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:11:46,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:46,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:46,924 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:46,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:46,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:47,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:49,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:51,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:51,455 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:51,456 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:51,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:51,461 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:52,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:11:53,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:11:56,461 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:11:59,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:11:59,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:11:59,889 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:11:59,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:11:59,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:11:59,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:11:59,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:00,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:01,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:02,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:04,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:04,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:04,129 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:04,134 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:05,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:05,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:08,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:08,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:08,789 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:08,789 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:08,794 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:09,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:09,384 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:12:11,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:13,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:13,449 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:13,450 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:13,451 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:13,456 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:14,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:14,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:14,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:14,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:15,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:17,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:17,882 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:17,882 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:17,887 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:18,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:18,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:19,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:23,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:23,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:23,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:23,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:24,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:24,773 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:25,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:29,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:29,886 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:29,887 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:29,887 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:29,892 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:29,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:29,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:29,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:30,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:31,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:35,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:35,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:35,965 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:35,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:35,970 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:36,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:37,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:39,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:12:40,385 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:43,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:43,136 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:43,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:43,142 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:44,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:44,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:44,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:44,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:45,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:46,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:48,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:48,415 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:48,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:48,421 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:49,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:49,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:51,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:53,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:12:53,826 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:12:53,826 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:12:53,831 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:12:54,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:12:55,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:12:56,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:12:59,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:12:59,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:12:59,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:12:59,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:00,228 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:00,228 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:00,233 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:01,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:01,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:02,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:05,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:05,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:05,210 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:05,216 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:06,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:07,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:08,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:09,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:13:09,481 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:09,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:09,482 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:09,487 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:10,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:11,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:13,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:13,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:13,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:13,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:13,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:14,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:14,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:14,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:14,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:15,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:19,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:19,080 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:19,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:19,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:19,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:19,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:21,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:24,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:24,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:24,122 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:24,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:24,127 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:24,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:25,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:29,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:29,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:29,833 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:29,834 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:29,839 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:29,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:29,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:29,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:30,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:31,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:34,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:34,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:34,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:34,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:34,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:35,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:37,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:39,391 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:13:39,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:39,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:39,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:39,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:39,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:40,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:41,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:44,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:44,693 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:44,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:44,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:44,698 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:44,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:44,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:13:45,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:47,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:49,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:49,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:49,505 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:49,510 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:50,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:50,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:51,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:54,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:54,951 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:54,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:54,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:55,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:13:55,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:13:57,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:13:59,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:13:59,408 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:13:59,408 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:13:59,413 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:13:59,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:13:59,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:13:59,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:00,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:01,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:01,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:04,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:04,301 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:04,301 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:04,306 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:05,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:05,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:06,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:09,390 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:14:09,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:09,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:09,581 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:09,586 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:10,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:11,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:11,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:14,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:14,435 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:14,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:14,440 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:14,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:14,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:14,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:15,193 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:15,193 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:17,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:18,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:18,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:18,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:18,943 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:19,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:21,199 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:22,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:24,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:24,077 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:24,077 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:24,083 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:24,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:25,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:28,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:28,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:28,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:28,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:28,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:29,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:29,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:29,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:29,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:29,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:33,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:33,147 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:33,147 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:33,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:33,153 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:33,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:35,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:38,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:38,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:38,955 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:38,956 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:38,960 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:39,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:39,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:14:41,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:43,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:44,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:44,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:44,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:14:47,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:47,578 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:47,579 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:47,584 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:48,225 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:48,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:49,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:53,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:53,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:53,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:53,014 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:53,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:54,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:55,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:58,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:14:58,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:14:58,222 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:14:58,227 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:14:58,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:14:59,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:14:59,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:14:59,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:14:59,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:14:59,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:04,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:04,074 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:04,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:04,079 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:04,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:05,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:05,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:09,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:09,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:09,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:09,193 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:09,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:09,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:15:10,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:11,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:14,611 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:14,612 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:14,612 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:14,617 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:14,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:14,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:14,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:15,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:15,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:16,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:21,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:21,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:21,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:21,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:21,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:22,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:23,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:25,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:25,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:25,945 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:25,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:26,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:27,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:27,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:29,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:29,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:29,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:32,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:32,071 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:32,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:32,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:32,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:33,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:33,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:37,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:37,215 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:37,215 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:37,221 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:37,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:38,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:39,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:39,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:15:42,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:42,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:42,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:42,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:42,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:43,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:44,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:44,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:44,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:15:46,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:46,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:46,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:46,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:47,286 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:49,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:49,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:52,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:52,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:52,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:52,421 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:53,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:53,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:55,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:15:58,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:15:58,387 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:15:58,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:15:58,392 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:15:59,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:15:59,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:15:59,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:15:59,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:15:59,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:01,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:03,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:03,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:03,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:03,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:03,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:05,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:06,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:08,105 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:08,106 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:08,106 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:08,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:08,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:09,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:09,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:16:11,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:14,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:14,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:14,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:14,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:14,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:14,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:14,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:14,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:15,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:17,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:20,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:20,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:20,166 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:20,172 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:20,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:21,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:23,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:25,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:25,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:25,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:25,649 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:26,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:27,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:28,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:29,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:29,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:29,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:30,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:30,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:30,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:30,220 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:30,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:31,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:34,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:35,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:35,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:35,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:35,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:36,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:37,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:39,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:16:39,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:40,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:40,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:40,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:40,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:40,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:41,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:44,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:44,047 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:44,047 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:44,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:44,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:44,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:16:45,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:49,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:49,468 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:49,468 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:49,473 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:50,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:50,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:51,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:55,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:16:56,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:16:56,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:16:56,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:16:56,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:16:57,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:16:57,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:16:59,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:16:59,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:16:59,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:01,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:01,171 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:01,172 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:01,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:01,177 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:01,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:03,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:05,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:05,277 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:05,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:05,282 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:05,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:06,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:07,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:09,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:09,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:09,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:09,298 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:09,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:09,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:17:11,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:11,405 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:13,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:13,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:13,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:13,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:14,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:14,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:14,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:14,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:15,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:17,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:18,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:18,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:18,179 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:18,183 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:18,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:19,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:22,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:27,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:27,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:27,703 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:27,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:27,708 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:28,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:29,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:29,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:29,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:29,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:32,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:33,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:33,062 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:33,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:33,067 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:33,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:35,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:37,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:37,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:37,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:37,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:37,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:38,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:39,406 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:17:39,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:42,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:42,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:42,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:42,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:42,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:43,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:43,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:44,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:44,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:17:46,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:46,726 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:46,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:46,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:47,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:47,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:48,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:51,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:51,007 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:51,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:51,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:51,436 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:53,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:54,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:55,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:17:55,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:17:55,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:17:55,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:17:56,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:17:57,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:17:59,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:17:59,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:17:59,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:17:59,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:00,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:00,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:00,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:00,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:01,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:03,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:04,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:05,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:05,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:05,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:05,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:06,452 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:07,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:09,411 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:18:10,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:11,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:11,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:11,518 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:11,523 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:12,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:13,461 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:14,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:14,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:14,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:16,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:16,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:16,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:16,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:16,223 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:16,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:17,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:21,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:23,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:23,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:23,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:23,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:23,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:25,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:27,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:27,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:27,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:27,647 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:27,652 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:28,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:29,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:29,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:29,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:29,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:32,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:32,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:32,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:32,071 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:32,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:33,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:33,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:38,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:39,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:18:40,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:40,531 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:40,531 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:40,536 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:41,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:41,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:43,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:44,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:44,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:18:45,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:45,605 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:45,606 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:45,611 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:46,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:47,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:48,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:53,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:53,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:53,819 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:53,819 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:53,824 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:54,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:55,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:18:58,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:18:59,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:18:59,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:18:59,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:18:59,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:18:59,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:18:59,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:18:59,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:18:59,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:01,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:04,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:05,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:05,072 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:05,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:05,077 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:05,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:07,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:09,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:09,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:09,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:09,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:09,415 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:19:09,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:09,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:11,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:14,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:14,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:14,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:14,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:14,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:14,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:14,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:14,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:15,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:15,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:20,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:20,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:20,260 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:20,261 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:20,266 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:20,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:21,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:25,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:25,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:25,417 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:25,418 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:25,423 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:25,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:27,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:29,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:29,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:29,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:31,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:32,522 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:32,523 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:32,523 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:32,528 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:32,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:33,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:36,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:38,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:38,280 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:38,280 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:38,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:38,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:39,414 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:19:39,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:42,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:42,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:42,972 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:42,973 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:42,979 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:43,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:43,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:44,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:44,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:44,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:19:48,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:48,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:48,999 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:48,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:49,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:49,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:51,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:54,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:57,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:19:57,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:19:57,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:19:57,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:19:57,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:19:59,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:19:59,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:19:59,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:19:59,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:19:59,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:02,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:02,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:02,992 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:02,997 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:03,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:03,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:04,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:09,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:09,021 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:09,021 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:09,027 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:09,417 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:20:09,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:10,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:11,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:13,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:13,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:13,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:13,780 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:14,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:14,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:14,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:14,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:15,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:16,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:18,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:18,900 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:18,900 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:18,905 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:19,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:19,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:21,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:24,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:24,335 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:24,335 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:24,340 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:24,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:25,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:27,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:28,822 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:28,822 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:28,823 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:28,828 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:29,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:29,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:29,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:29,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:29,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:33,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:33,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:33,112 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:33,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:33,117 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:33,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:35,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:37,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:37,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:37,544 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:37,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:37,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:38,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:39,419 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:20:39,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:44,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:44,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:44,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:44,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:44,310 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:44,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:45,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:20:45,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:20:45,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:20:45,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:49,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:49,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:49,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:49,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:49,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:50,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:51,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:53,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:53,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:53,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:53,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:53,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:55,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:20:55,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:20:58,294 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:20:58,295 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:20:58,295 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:20:58,301 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:20:58,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:20:59,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:00,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:00,659 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:00,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:00,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:03,258 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:03,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:03,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:03,264 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:03,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:05,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:06,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:08,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:08,932 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:08,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:08,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:09,421 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:21:09,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:09,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:11,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:15,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:15,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:16,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:16,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:16,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:16,065 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:16,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:17,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:17,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:20,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:20,862 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:20,862 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:20,867 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:21,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:21,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:22,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:26,254 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:26,255 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:26,255 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:26,260 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:26,778 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:27,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:28,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:30,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:30,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:30,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:31,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:31,636 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:31,637 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:31,642 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:31,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:33,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:33,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:38,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:38,842 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:38,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:38,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:38,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:39,427 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:21:39,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:39,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:44,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:44,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:44,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:44,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:44,279 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:44,796 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:45,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:21:45,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:21:45,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:21:45,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:49,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:52,557 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:52,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:52,558 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:52,563 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:52,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:21:53,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:55,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:21:58,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:21:58,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:21:58,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:21:58,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:21:59,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:21:59,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:00,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:05,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:05,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:05,251 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:05,256 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:05,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:06,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:07,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:09,425 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:22:09,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:09,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:09,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:09,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:09,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:11,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:11,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:14,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:14,489 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:14,490 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:14,495 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:14,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:15,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:15,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:15,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:16,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:19,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:19,044 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:19,044 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:19,049 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:19,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:19,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:22,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:24,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:24,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:24,275 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:24,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:24,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:25,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:27,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:28,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:28,773 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:28,773 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:28,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:28,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:29,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:30,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:30,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:30,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:32,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:32,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:32,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:32,990 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:32,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:33,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:33,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:37,118 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:37,118 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:37,119 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:37,124 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:37,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:37,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:38,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:39,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:22:43,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:45,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:22:45,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:22:45,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:22:45,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:45,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:45,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:45,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:46,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:47,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:48,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:50,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:50,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:50,196 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:50,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:50,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:51,865 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:54,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:22:54,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:54,790 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:54,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:54,796 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:54,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:22:55,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:59,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:22:59,012 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:22:59,012 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:22:59,017 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:22:59,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:22:59,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:00,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:00,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:00,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:03,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:03,456 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:03,456 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:03,461 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:03,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:05,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:05,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:09,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:23:09,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:09,517 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:09,518 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:09,522 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:09,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:10,523 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:11,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:13,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:13,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:13,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:13,908 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:14,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:15,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:15,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:15,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:20,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:22,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:22,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:22,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:22,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:22,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:23,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:26,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:26,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:26,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:26,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:26,722 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:26,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:27,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:30,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:30,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:30,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:30,823 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:30,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:30,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:30,927 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:31,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:31,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:31,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:35,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:35,124 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:35,124 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:35,129 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:35,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:35,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:37,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:39,431 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:23:42,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:43,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:43,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:43,166 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:43,174 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:43,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:43,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:45,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:23:45,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:23:45,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:23:47,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:51,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:51,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:51,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:51,742 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:51,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:53,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:23:53,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:56,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:23:56,151 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:23:56,151 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:23:56,157 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:23:56,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:23:57,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:23:59,157 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:00,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:01,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:01,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:01,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:01,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:01,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:01,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:04,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:05,461 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:05,461 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:05,462 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:05,467 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:05,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:07,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:09,437 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:24:09,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:09,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:09,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:09,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:09,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:10,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:12,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:14,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:15,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:16,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:16,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:16,872 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:16,877 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:17,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:18,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:20,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:21,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:21,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:21,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:21,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:22,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:24,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:25,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:25,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:25,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:25,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:25,974 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:26,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:28,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:30,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:30,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:30,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:31,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:31,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:31,351 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:31,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:31,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:32,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:34,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:36,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:36,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:36,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:36,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:36,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:37,031 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:38,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:39,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:24:40,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:40,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:40,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:40,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:41,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:41,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:42,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:45,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:24:45,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:24:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:24:47,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:49,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:49,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:49,286 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:49,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:50,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:50,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:53,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:24:53,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:24:53,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:24:53,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:24:53,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:24:54,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:24:56,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:24:58,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:00,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:00,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:00,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:00,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:00,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:00,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:00,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:01,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:02,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:03,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:05,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:05,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:05,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:05,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:06,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:06,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:09,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:09,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:25:09,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:09,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:09,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:09,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:10,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:12,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:14,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:14,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:14,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:14,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:14,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:15,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:15,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:15,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:15,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:16,073 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:19,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:20,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:20,531 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:20,531 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:20,536 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:21,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:22,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:25,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:29,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:29,112 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:29,112 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:29,117 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:30,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:30,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:30,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:35,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:35,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:35,321 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:35,326 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:36,106 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:36,106 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:36,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:39,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:25:39,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:39,752 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:39,752 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:39,757 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:40,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:41,758 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:42,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:45,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:45,123 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:45,123 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:45,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:45,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:25:45,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:25:45,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:25:46,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:46,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:46,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:49,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:49,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:49,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:49,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:50,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:52,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:52,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:25:53,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:25:53,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:25:53,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:25:53,774 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:25:54,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:25:56,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:25:57,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:00,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:00,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:00,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:00,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:00,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:00,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:00,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:00,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:02,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:03,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:03,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:03,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:03,916 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:03,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:04,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:06,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:08,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:09,445 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:26:10,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:10,005 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:10,006 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:10,011 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:10,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:12,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:14,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:15,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:15,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:15,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:16,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:16,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:16,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:16,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:17,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:18,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:19,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:24,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:24,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:24,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:24,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:25,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:25,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:26,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:28,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:28,981 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:28,981 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:28,986 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:29,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:30,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:30,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:30,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:30,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:30,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:33,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:33,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:33,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:33,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:34,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:34,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:36,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:37,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:37,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:37,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:37,241 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:38,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:38,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:39,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:26:41,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:42,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:42,663 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:42,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:42,668 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:43,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:44,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:45,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:26:45,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:26:45,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:26:46,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:47,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:47,978 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:47,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:47,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:48,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:50,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:52,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:26:53,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:26:53,093 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:26:53,093 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:26:53,098 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:26:53,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:26:54,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:26:57,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:00,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:00,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:00,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:02,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:02,124 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:02,125 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:02,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:02,130 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:02,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:04,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:07,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:09,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:27:10,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:10,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:10,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:10,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:11,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:12,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:12,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:15,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:15,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:15,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:16,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:16,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:16,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:16,437 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:17,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:17,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:18,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:21,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:21,713 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:21,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:21,719 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:22,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:22,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:24,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:25,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:25,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:25,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:25,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:26,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:27,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:28,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:29,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:29,935 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:29,935 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:29,940 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:30,243 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:30,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:30,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:32,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:32,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:34,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:34,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:34,281 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:34,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:35,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:36,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:38,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:39,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:27:42,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:42,717 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:42,717 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:42,723 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:43,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:43,724 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:44,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:45,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:27:45,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:27:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:27:46,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:46,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:46,950 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:46,955 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:47,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:48,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:48,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:53,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:55,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:55,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:55,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:55,063 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:27:55,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:27:56,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:27:59,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:27:59,493 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:27:59,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:27:59,494 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:27:59,499 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:00,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:00,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:00,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:00,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:00,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:04,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:08,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:08,387 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:08,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:08,393 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:09,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:09,450 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:28:10,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:10,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:12,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:12,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:12,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:12,762 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:13,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:14,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:15,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:15,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:15,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:15,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:17,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:17,157 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:17,157 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:17,162 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:17,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:18,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:21,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:22,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:22,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:22,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:22,041 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:22,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:24,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:26,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:26,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:26,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:26,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:26,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:27,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:28,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:30,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:31,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:31,417 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:31,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:31,422 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:32,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:32,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:32,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:37,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:39,456 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:28:39,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:39,620 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:39,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:39,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:40,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:42,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:42,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:28:45,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:28:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:28:45,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:45,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:45,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:45,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:46,350 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:47,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:48,354 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:50,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:50,012 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:50,012 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:50,018 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:50,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:52,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:53,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:55,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:55,057 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:55,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:55,062 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:55,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:28:56,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:28:58,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:28:59,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:28:59,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:28:59,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:28:59,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:28:59,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:00,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:00,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:00,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:00,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:03,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:03,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:03,582 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:03,582 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:03,588 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:04,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:04,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:08,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:08,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:08,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:08,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:08,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:09,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:09,455 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:29:10,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:12,243 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:12,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:12,244 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:12,249 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:12,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:14,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:14,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:15,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:15,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:19,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:19,689 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:19,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:19,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:19,694 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:20,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:22,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:24,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:24,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:24,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:24,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:24,698 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:25,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:26,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:29,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:29,411 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:29,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:29,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:30,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:30,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:30,417 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:30,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:33,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:33,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:33,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:33,690 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:34,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:35,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:36,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:37,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:37,942 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:37,942 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:37,948 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:38,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:39,460 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:29:40,413 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:41,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:42,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:42,433 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:42,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:42,438 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:43,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:44,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:29:45,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:29:45,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:29:46,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:47,750 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:47,751 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:47,751 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:47,756 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:48,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:50,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:52,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:53,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:53,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:53,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:53,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:53,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:29:54,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:29:58,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:29:58,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:29:58,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:29:58,165 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:29:58,171 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:29:58,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:00,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:00,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:00,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:00,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:03,554 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:03,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:03,555 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:03,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:03,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:04,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:04,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:08,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:08,272 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:08,273 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:08,278 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:08,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:09,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:09,459 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:30:10,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:12,759 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:12,760 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:12,761 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:12,766 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:13,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:14,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:14,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:15,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:15,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:15,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:19,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:21,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:21,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:21,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:21,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:21,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:22,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:25,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:29,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:29,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:29,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:29,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:29,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:30,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:30,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:30,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:30,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:30,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:34,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:34,197 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:34,197 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:34,202 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:34,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:36,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:36,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:38,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:38,568 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:38,568 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:38,573 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:39,461 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:30:39,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:40,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:41,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:44,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:44,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:44,323 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:44,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:44,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:45,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:30:45,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:30:45,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:30:46,483 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:46,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:48,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:48,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:48,938 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:48,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:49,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:50,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:51,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:53,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:53,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:53,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:53,684 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:54,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:30:54,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:30:57,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:30:58,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:30:58,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:30:58,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:30:58,701 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:30:59,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:00,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:00,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:00,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:00,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:02,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:03,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:03,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:03,126 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:03,130 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:03,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:04,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:08,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:08,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:08,264 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:08,264 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:08,269 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:08,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:09,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:31:10,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:12,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:12,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:12,658 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:12,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:13,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:13,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:14,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:15,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:15,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:15,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:18,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:18,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:18,006 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:18,010 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:18,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:19,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:20,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:23,227 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:23,228 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:23,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:23,233 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:23,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:24,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:24,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:27,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:27,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:27,584 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:27,589 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:28,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:28,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:29,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:30,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:30,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:30,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:34,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:34,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:34,995 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:34,995 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:35,000 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:35,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:36,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:39,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:31:40,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:41,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:41,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:41,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:41,472 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:41,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:42,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:45,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:45,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:31:45,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:31:45,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:31:50,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:31:52,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:31:52,020 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:31:52,020 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:31:52,025 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:31:52,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:31:54,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:31:56,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:00,525 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:00,526 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:00,526 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:00,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:00,557 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:00,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:00,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:00,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:01,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:02,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:06,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:06,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:06,581 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:06,586 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:07,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:07,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:08,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:09,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:32:11,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:11,247 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:11,247 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:11,252 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:11,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:12,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:13,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:15,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:15,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:15,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:16,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:16,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:16,532 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:16,537 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:16,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:18,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:18,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:21,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:21,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:21,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:21,393 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:21,579 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:22,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:24,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:26,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:26,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:26,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:26,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:27,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:28,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:29,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:30,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:30,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:30,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:31,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:31,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:31,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:31,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:32,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:34,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:35,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:39,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:32:39,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:39,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:39,637 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:39,642 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:40,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:40,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:40,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:44,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:44,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:44,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:44,269 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:44,601 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:45,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:32:45,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:32:45,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:32:45,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:46,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:50,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:50,021 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:50,021 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:50,026 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:50,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:51,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:52,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:53,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:54,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:54,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:54,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:54,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:32:56,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:32:57,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:32:58,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:32:58,537 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:32:58,538 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:32:58,543 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:32:58,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:00,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:00,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:00,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:02,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:05,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:05,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:05,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:05,057 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:05,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:06,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:08,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:09,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:33:09,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:09,589 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:09,590 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:09,595 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:09,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:10,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:13,595 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:15,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:15,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:18,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:18,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:18,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:18,043 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:18,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:19,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:20,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:22,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:22,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:22,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:22,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:23,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:24,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:24,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:28,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:28,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:28,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:28,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:28,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:30,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:30,663 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:30,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:30,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:30,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:32,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:32,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:32,998 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:33,003 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:33,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:34,668 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:36,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:38,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:38,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:38,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:38,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:38,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:39,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:33:40,671 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:41,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:45,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:33:45,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:33:45,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:33:46,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:46,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:46,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:46,102 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:46,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:47,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:48,678 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:52,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:52,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:52,196 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:52,196 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:52,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:52,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:33:54,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:33:57,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:33:58,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:33:58,894 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:33:58,894 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:33:58,899 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:33:59,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:00,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:00,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:00,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:00,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:02,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:03,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:03,748 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:03,749 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:03,753 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:04,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:04,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:08,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:08,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:08,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:08,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:08,219 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:08,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:09,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:34:10,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:12,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:12,668 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:12,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:12,673 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:12,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:13,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:14,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:15,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:18,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:22,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:22,327 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:22,328 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:22,334 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:22,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:24,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:24,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:26,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:26,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:26,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:26,711 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:26,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:28,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:29,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:30,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:30,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:31,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:31,441 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:31,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:31,446 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:31,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:32,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:35,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:35,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:35,974 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:35,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:35,979 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:36,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:36,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:39,481 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:34:41,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:42,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:42,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:42,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:42,014 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:42,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:42,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:45,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:34:45,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:34:45,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:34:46,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:46,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:46,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:46,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:46,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:47,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:48,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:51,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:51,108 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:51,108 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:51,113 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:51,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:52,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:53,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:34:55,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:34:55,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:34:55,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:34:55,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:34:56,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:34:56,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:34:58,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:00,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:00,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:00,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:01,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:01,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:01,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:01,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:02,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:02,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:04,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:08,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:08,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:08,004 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:08,009 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:08,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:08,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:09,480 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:35:09,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:13,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:13,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:13,029 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:13,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:13,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:14,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:15,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:15,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:18,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:18,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:18,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:18,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:18,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:20,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:20,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:23,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:23,236 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:23,237 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:23,242 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:23,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:24,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:26,243 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:27,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:27,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:27,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:27,700 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:27,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:28,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:30,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:30,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:30,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:31,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:32,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:32,920 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:32,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:32,925 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:33,790 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:34,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:36,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:37,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:37,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:37,346 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:37,383 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:37,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:38,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:39,485 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:35:42,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:43,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:43,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:43,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:43,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:43,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:44,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:45,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:35:45,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:35:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:35:47,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:49,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:49,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:49,194 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:49,199 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:49,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:50,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:53,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:35:53,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:53,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:53,782 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:53,787 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:53,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:54,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:35:58,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:35:58,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:35:58,210 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:35:58,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:35:58,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:35:59,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:00,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:00,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:00,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:02,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:02,676 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:02,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:02,682 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:02,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:04,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:04,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:07,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:07,108 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:07,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:07,113 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:07,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:08,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:09,485 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:36:10,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:11,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:11,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:11,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:11,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:11,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:12,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:15,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:15,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:15,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:15,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:18,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:18,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:18,474 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:18,479 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:18,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:20,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:21,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:23,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:23,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:23,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:23,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:24,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:24,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:26,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:30,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:30,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:31,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:32,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:32,308 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:32,308 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:32,313 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:32,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:34,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:36,845 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:36,846 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:36,846 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:36,851 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:36,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:37,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:38,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:39,487 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:36:41,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:41,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:41,366 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:41,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:41,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:42,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:43,373 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:45,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:45,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:45,544 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:45,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:45,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:36:45,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:36:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:36:45,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:46,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:48,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:51,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:51,742 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:51,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:51,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:51,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:52,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:36:54,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:36:57,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:36:57,204 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:36:57,204 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:36:57,210 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:36:57,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:36:58,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:00,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:00,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:01,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:01,483 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:01,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:01,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:01,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:02,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:05,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:08,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:08,153 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:08,154 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:08,158 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:08,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:08,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:09,489 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:37:11,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:13,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:13,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:13,784 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:13,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:13,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:14,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:15,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:15,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:15,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:16,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:18,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:18,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:18,265 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:18,270 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:18,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:20,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:22,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:24,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:24,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:24,547 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:24,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:24,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:26,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:27,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:29,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:29,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:29,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:29,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:29,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:30,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:30,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:30,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:30,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:32,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:33,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:33,186 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:33,186 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:33,191 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:33,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:34,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:38,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:38,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:38,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:38,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:38,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:38,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:38,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:39,491 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:37:43,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:43,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:43,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:43,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:43,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:44,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:44,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:45,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:37:45,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:37:45,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:37:49,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:49,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:49,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:49,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:49,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:49,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:50,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:54,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:55,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:55,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:55,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:55,030 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:55,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:37:56,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:37:59,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:37:59,530 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:37:59,531 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:37:59,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:37:59,537 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:37:59,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:00,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:00,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:04,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:04,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:04,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:04,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:04,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:05,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:06,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:09,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:38:10,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:13,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:13,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:13,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:13,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:13,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:14,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:15,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:15,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:15,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:16,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:17,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:17,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:17,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:17,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:17,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:18,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:21,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:21,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:21,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:21,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:21,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:21,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:22,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:26,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:26,071 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:26,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:26,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:26,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:26,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:27,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:30,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:30,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:30,576 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:30,582 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:30,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:30,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:30,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:30,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:32,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:32,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:37,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:39,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:39,027 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:39,028 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:39,033 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:39,495 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:38:39,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:40,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:43,497 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:43,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:43,951 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:43,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:43,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:43,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:44,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:45,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:38:45,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:38:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:38:48,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:48,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:48,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:48,322 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:48,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:49,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:51,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:52,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:52,684 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:52,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:52,689 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:53,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:54,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:38:55,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:56,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:38:56,968 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:38:56,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:38:56,974 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:38:57,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:38:59,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:38:59,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:00,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:00,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:00,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:01,594 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:01,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:01,595 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:01,600 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:02,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:03,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:05,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:06,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:06,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:06,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:06,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:07,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:07,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:09,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:39:10,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:10,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:10,447 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:10,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:11,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:11,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:13,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:15,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:15,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:15,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:16,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:16,445 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:16,445 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:16,450 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:16,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:17,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:19,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:21,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:25,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:25,593 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:25,594 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:25,599 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:26,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:26,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:27,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:30,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:30,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:30,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:30,961 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:30,966 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:31,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:31,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:33,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:35,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:35,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:35,135 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:35,140 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:36,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:37,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:37,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:39,500 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:39:39,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:39,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:39,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:39,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:40,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:41,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:42,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:39:45,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:39:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:39:47,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:48,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:48,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:48,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:48,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:49,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:51,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:53,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:39:54,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:39:54,182 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:39:54,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:39:54,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:39:55,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:39:55,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:39:59,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:00,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:00,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:00,357 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:00,362 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:00,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:00,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:00,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:01,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:01,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:04,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:04,837 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:04,837 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:04,837 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:04,842 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:05,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:07,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:08,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:08,963 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:08,963 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:08,969 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:09,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:09,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:40:10,506 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:11,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:13,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:13,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:13,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:13,655 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:14,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:15,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:15,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:15,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:15,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:15,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:18,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:18,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:18,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:18,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:19,106 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:19,107 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:21,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:24,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:24,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:24,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:24,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:25,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:25,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:26,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:30,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:30,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:30,478 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:30,483 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:30,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:31,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:31,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:33,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:36,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:38,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:38,814 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:38,814 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:38,819 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:39,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:39,504 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:40:41,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:42,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:43,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:43,047 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:43,047 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:43,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:43,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:45,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:40:45,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:40:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:40:47,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:49,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:49,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:49,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:49,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:50,134 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:51,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:53,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:40:53,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:40:53,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:40:53,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:40:53,870 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:40:54,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:40:55,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:40:58,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:00,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:00,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:00,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:02,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:02,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:02,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:02,268 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:03,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:03,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:04,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:09,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:09,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:09,018 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:09,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:09,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:09,506 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:41:09,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:11,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:12,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:12,835 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:12,835 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:12,841 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:13,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:14,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:15,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:15,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:15,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:18,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:18,777 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:18,777 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:18,782 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:19,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:20,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:21,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:25,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:27,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:27,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:27,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:27,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:28,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:29,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:30,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:30,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:30,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:30,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:31,585 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:31,586 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:31,586 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:31,591 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:32,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:33,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:36,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:36,103 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:36,103 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:36,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:36,108 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:36,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:37,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:39,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:41:40,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:40,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:40,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:40,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:40,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:41,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:41,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:45,149 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:45,150 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:45,150 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:45,155 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:45,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:45,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:41:45,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:41:45,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:41:46,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:47,232 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:51,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:51,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:51,469 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:51,474 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:52,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:52,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:41:53,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:55,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:41:55,628 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:41:55,628 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:41:55,634 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:41:56,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:41:57,243 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:41:57,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:00,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:00,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:00,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:00,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:00,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:00,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:00,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:01,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:01,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:02,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:07,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:08,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:08,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:08,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:08,879 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:09,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:09,510 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:42:11,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:13,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:15,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:15,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:17,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:17,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:17,262 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:17,267 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:17,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:19,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:19,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:21,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:21,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:21,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:21,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:22,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:23,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:24,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:25,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:25,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:25,818 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:25,824 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:26,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:27,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:29,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:30,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:30,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:30,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:30,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:30,920 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:30,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:30,926 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:31,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:33,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:34,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:35,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:35,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:35,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:35,322 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:36,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:37,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:39,515 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:42:40,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:40,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:40,972 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:40,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:40,978 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:41,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:43,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:45,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:45,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:45,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:45,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:45,618 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:45,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:42:45,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:42:45,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:42:46,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:47,325 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:49,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:49,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:49,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:49,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:50,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:50,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:51,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:42:55,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:42:58,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:42:58,296 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:42:58,296 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:42:58,301 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:42:58,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:42:59,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:00,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:00,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:00,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:02,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:02,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:02,628 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:02,633 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:03,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:05,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:06,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:09,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:43:09,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:09,672 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:09,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:09,677 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:10,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:11,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:11,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:15,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:15,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:15,174 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:15,179 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:15,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:15,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:15,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:16,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:17,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:20,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:20,252 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:20,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:20,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:20,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:21,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:22,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:25,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:25,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:25,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:25,279 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:25,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:27,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:28,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:30,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:30,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:30,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:31,762 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:31,763 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:31,763 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:31,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:32,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:33,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:33,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:38,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:38,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:38,342 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:38,347 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:38,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:39,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:39,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:39,519 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:43:44,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:44,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:44,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:44,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:44,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:45,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:45,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:45,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:43:45,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:43:45,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:43:49,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:51,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:51,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:51,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:51,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:51,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:43:53,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:55,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:43:56,498 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:43:56,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:43:56,499 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:43:56,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:43:57,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:43:57,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:00,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:00,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:00,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:00,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:02,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:02,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:02,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:02,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:02,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:03,413 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:06,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:07,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:07,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:07,286 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:07,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:07,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:09,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:09,519 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:44:11,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:12,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:12,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:12,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:12,030 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:12,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:13,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:15,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:15,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:15,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:16,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:17,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:17,663 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:17,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:17,668 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:18,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:19,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:22,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:23,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:23,004 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:23,005 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:23,010 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:23,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:25,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:27,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:27,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:27,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:27,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:28,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:28,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:29,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:30,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:30,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:30,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:32,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:32,657 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:32,658 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:32,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:33,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:33,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:33,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:38,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:38,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:38,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:38,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:38,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:39,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:39,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:39,521 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:44:44,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:44,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:44,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:44,017 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:44,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:45,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:45,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:45,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:44:45,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:44:45,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:44:49,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:49,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:49,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:49,776 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:50,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:50,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:44:51,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:54,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:44:54,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:44:54,258 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:44:54,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:44:54,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:44:55,508 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:44:56,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:00,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:00,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:00,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:00,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:00,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:00,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:00,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:01,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:01,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:03,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:05,094 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:05,094 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:05,094 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:05,099 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:05,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:07,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:07,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:09,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:09,227 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:09,227 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:09,232 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:09,523 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:45:09,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:11,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:12,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:14,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:14,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:14,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:14,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:15,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:15,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:15,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:17,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:20,207 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:20,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:20,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:20,213 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:20,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:21,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:23,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:24,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:24,596 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:24,596 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:24,601 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:25,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:25,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:28,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:30,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:30,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:30,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:33,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:33,153 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:33,153 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:33,158 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:33,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:34,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:35,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:38,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:38,068 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:38,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:38,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:38,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:39,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:45:39,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:39,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:42,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:42,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:42,444 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:42,449 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:42,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:43,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:45,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:45,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:45:45,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:45:45,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:45:46,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:46,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:46,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:46,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:47,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:47,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:50,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:51,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:51,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:51,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:51,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:52,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:53,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:45:56,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:45:56,197 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:45:56,197 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:45:56,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:45:56,202 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:45:56,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:45:57,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:00,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:00,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:00,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:01,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:02,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:02,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:02,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:02,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:02,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:03,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:06,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:06,620 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:06,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:06,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:07,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:07,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:07,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:09,530 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:46:13,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:13,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:13,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:13,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:13,172 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:13,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:15,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:15,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:15,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:18,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:19,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:19,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:19,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:19,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:19,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:21,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:24,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:25,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:25,945 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:25,945 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:25,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:26,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:27,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:29,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:30,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:30,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:30,564 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:30,569 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:30,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:30,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:30,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:30,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:31,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:35,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:35,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:35,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:35,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:35,487 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:35,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:37,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:39,529 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:46:39,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:39,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:39,883 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:39,888 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:40,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:40,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:41,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:45,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:46:45,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:46:45,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:46:45,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:46,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:46,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:46,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:46,104 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:46,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:47,651 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:50,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:50,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:50,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:50,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:51,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:51,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:53,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:46:55,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:46:55,305 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:46:55,305 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:46:55,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:46:55,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:46:57,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:46:57,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:00,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:00,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:00,202 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:00,207 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:00,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:00,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:00,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:01,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:02,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:07,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:07,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:07,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:07,400 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:07,671 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:08,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:09,531 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:47:09,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:13,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:13,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:13,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:13,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:13,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:14,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:15,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:15,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:15,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:18,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:18,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:18,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:18,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:18,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:19,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:19,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:21,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:21,983 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:21,984 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:21,988 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:22,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:23,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:24,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:29,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:30,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:30,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:30,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:30,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:30,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:30,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:30,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:31,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:33,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:35,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:39,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:39,464 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:39,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:39,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:39,533 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:47:39,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:41,534 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:41,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:44,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:44,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:44,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:44,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:45,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:47:45,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:47:45,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:47:45,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:45,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:46,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:50,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:50,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:50,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:50,530 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:50,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:51,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:52,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:47:56,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:47:56,749 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:47:56,749 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:47:56,754 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:47:57,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:47:57,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:47:57,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:00,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:00,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:01,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:01,437 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:01,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:01,442 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:01,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:03,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:03,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:06,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:06,327 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:06,328 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:06,333 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:06,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:07,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:09,333 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:09,535 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:48:11,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:11,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:11,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:11,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:11,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:13,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:15,051 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:15,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:15,477 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:15,477 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:15,483 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:15,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:15,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:15,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:15,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:17,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:20,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:21,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:21,141 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:21,141 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:21,146 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:21,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:23,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:25,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:25,629 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:25,629 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:25,635 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:25,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:26,635 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:27,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:30,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:30,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:30,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:30,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:30,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:30,975 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:30,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:31,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:32,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:33,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:36,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:36,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:36,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:36,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:36,763 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:37,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:37,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:39,540 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:48:41,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:41,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:41,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:41,929 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:42,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:42,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:43,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:45,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:48:45,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:48:45,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:48:47,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:50,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:50,305 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:50,305 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:50,310 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:50,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:51,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:53,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:54,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:54,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:54,180 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:54,185 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:54,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:55,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:48:58,730 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:48:58,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:48:58,731 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:48:58,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:48:58,736 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:48:58,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:48:59,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:00,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:00,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:00,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:03,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:04,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:04,622 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:04,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:04,627 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:04,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:05,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:09,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:49:09,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:13,010 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:13,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:13,011 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:13,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:13,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:13,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:15,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:15,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:15,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:20,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:24,374 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:24,375 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:24,375 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:24,380 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:24,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:25,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:26,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:29,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:29,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:29,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:29,401 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:29,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:30,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:30,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:30,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:31,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:31,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:34,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:34,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:34,835 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:34,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:35,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:35,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:37,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:39,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:49:40,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:40,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:40,211 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:40,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:40,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:41,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:43,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:45,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:49:45,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:49:45,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:49:48,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:48,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:48,575 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:48,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:48,580 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:48,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:49,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:53,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:49:56,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:49:56,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:49:56,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:49:56,782 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:49:56,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:49:57,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:49:58,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:00,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:00,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:00,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:03,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:03,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:03,782 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:03,787 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:03,787 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:03,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:05,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:08,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:08,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:08,847 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:08,847 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:08,852 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:08,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:09,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:50:09,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:14,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:14,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:14,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:14,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:14,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:14,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:15,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:15,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:15,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:19,082 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:19,083 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:19,083 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:19,088 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:19,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:20,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:21,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:24,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:24,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:24,269 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:24,274 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:24,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:25,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:25,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:28,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:28,778 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:28,779 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:28,784 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:28,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:29,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:30,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:30,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:30,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:30,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:33,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:33,869 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:33,869 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:33,874 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:33,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:35,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:36,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:39,546 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:50:42,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:42,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:42,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:42,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:42,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:42,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:43,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:45,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:50:45,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:50:45,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:50:46,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:46,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:46,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:46,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:47,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:47,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:48,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:53,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:53,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:53,460 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:53,460 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:53,465 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:53,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:55,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:50:58,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:50:58,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:50:58,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:50:58,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:50:58,875 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:50:58,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:50:59,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:00,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:00,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:03,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:04,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:04,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:04,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:04,142 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:04,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:05,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:08,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:08,732 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:08,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:08,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:08,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:09,548 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:51:09,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:09,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:14,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:14,457 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:14,458 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:14,463 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:14,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:15,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:15,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:15,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:15,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:19,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:19,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:19,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:19,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:19,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:20,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:21,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:24,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:24,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:24,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:24,900 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:24,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:25,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:25,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:29,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:29,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:29,744 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:29,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:29,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:30,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:30,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:30,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:31,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:31,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:34,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:34,020 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:34,020 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:34,025 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:34,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:35,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:37,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:38,866 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:38,867 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:38,867 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:38,873 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:38,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:39,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:51:39,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:42,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:43,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:43,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:43,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:43,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:43,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:45,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:51:45,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:51:45,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:51:45,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:47,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:48,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:48,060 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:48,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:48,065 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:48,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:49,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:53,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:51:55,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:51:55,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:51:55,251 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:51:55,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:51:55,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:51:57,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:51:58,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:00,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:00,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:00,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:00,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:00,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:00,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:00,983 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:01,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:03,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:05,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:05,679 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:05,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:05,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:05,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:07,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:09,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:52:09,558 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:10,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:10,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:10,463 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:10,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:11,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:12,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:15,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:15,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:15,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:15,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:15,231 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:15,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:15,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:16,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:16,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:20,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:21,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:21,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:21,723 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:21,730 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:22,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:24,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:26,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:26,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:26,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:26,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:26,230 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:27,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:28,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:30,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:30,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:30,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:30,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:30,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:30,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:30,946 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:31,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:31,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:32,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:36,105 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:36,105 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:36,106 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:36,113 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:37,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:37,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:38,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:39,554 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:52:40,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:40,350 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:40,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:40,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:41,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:42,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:42,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:45,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:45,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:45,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:45,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:45,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:52:45,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:52:45,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:52:46,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:46,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:47,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:49,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:49,256 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:49,256 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:49,262 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:50,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:52,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:53,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:52:53,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:53,508 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:53,508 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:53,513 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:54,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:56,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:52:57,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:52:57,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:52:57,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:52:57,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:52:58,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:52:58,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:00,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:00,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:00,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:03,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:03,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:03,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:03,490 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:04,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:04,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:06,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:07,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:07,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:07,521 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:07,529 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:08,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:09,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:09,559 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:53:10,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:11,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:11,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:11,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:11,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:12,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:14,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:14,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:15,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:15,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:15,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:19,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:19,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:19,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:19,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:19,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:20,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:20,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:23,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:23,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:23,268 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:23,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:24,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:24,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:25,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:30,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:30,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:30,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:30,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:31,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:31,615 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:31,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:31,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:32,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:34,125 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:35,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:35,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:35,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:35,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:35,639 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:36,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:38,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:39,558 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:53:40,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:40,540 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:40,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:40,547 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:41,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:41,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:42,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:45,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:45,089 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:45,089 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:45,096 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:45,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:45,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:53:45,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:53:45,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:53:46,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:46,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:49,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:49,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:49,834 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:49,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:50,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:52,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:52,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:54,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:54,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:54,479 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:54,484 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:55,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:53:56,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:53:58,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:53:58,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:53:58,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:53:58,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:53:58,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:53:59,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:00,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:00,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:00,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:00,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:02,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:02,912 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:02,913 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:02,925 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:03,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:03,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:04,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:08,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:08,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:08,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:08,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:09,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:09,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:54:09,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:10,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:13,311 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:13,311 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:13,312 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:13,319 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:14,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:14,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:15,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:15,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:15,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:15,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:18,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:18,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:18,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:18,236 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:19,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:20,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:21,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:22,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:22,618 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:22,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:22,624 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:23,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:24,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:26,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:27,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:27,225 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:27,225 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:27,230 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:28,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:28,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:30,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:30,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:31,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:32,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:32,080 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:32,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:32,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:32,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:34,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:37,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:37,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:37,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:37,977 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:37,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:38,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:39,563 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:54:40,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:42,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:42,545 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:42,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:42,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:42,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:43,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:44,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:45,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:54:45,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:54:45,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:54:46,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:46,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:46,869 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:46,875 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:47,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:47,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:48,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:51,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:51,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:51,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:51,687 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:52,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:53,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:54:54,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:54:57,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:54:57,825 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:54:57,826 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:54:57,831 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:54:58,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:54:58,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:00,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:00,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:00,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:01,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:01,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:01,842 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:01,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:02,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:03,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:04,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:07,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:07,008 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:07,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:07,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:07,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:08,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:09,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:09,565 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:55:11,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:11,270 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:11,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:11,276 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:12,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:12,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:14,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:15,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:15,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:19,865 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:19,866 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:19,866 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:19,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:19,871 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:20,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:22,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:24,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:24,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:24,994 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:24,994 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:25,000 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:25,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:26,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:29,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:29,683 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:29,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:29,689 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:30,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:30,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:30,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:30,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:32,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:35,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:35,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:35,334 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:35,340 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:36,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:36,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:36,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:39,567 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:55:39,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:39,763 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:39,763 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:39,769 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:40,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:41,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:42,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:44,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:44,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:44,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:44,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:44,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:45,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:55:45,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:55:45,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:55:46,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:46,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:48,730 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:48,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:48,731 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:48,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:49,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:50,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:52,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:55:53,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:53,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:53,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:53,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:54,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:55:54,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:57,757 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:55:57,758 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:55:57,758 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:55:57,763 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:55:58,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:55:58,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:00,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:00,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:00,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:01,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:01,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:01,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:01,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:02,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:04,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:04,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:09,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:56:09,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:09,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:09,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:09,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:09,925 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:10,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:12,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:14,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:15,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:15,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:15,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:15,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:15,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:15,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:15,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:16,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:18,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:20,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:21,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:21,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:21,466 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:21,501 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:22,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:22,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:26,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:26,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:26,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:26,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:26,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:26,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:28,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:30,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:30,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:30,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:30,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:30,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:30,972 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:31,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:31,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:32,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:36,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:36,740 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:36,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:36,745 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:37,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:37,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:38,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:39,574 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:56:41,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:41,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:41,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:41,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:41,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:42,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:43,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:45,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:56:45,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:56:45,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:56:48,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:48,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:48,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:48,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:48,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:49,377 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:50,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:54,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:56:54,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:56:54,407 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:56:54,407 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:56:54,412 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:56:55,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:56:56,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:56:59,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:00,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:00,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:00,331 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:00,336 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:00,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:00,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:00,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:02,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:04,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:04,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:04,955 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:04,956 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:04,961 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:05,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:06,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:09,573 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:57:10,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:11,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:11,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:11,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:11,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:12,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:14,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:15,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:15,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:15,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:16,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:16,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:16,023 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:16,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:16,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:18,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:20,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:20,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:20,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:20,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:21,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:21,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:22,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:25,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:25,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:25,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:25,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:25,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:26,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:27,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:30,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:30,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:30,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:30,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:30,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:30,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:30,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:31,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:32,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:32,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:36,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:36,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:36,864 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:36,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:37,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:38,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:38,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:39,578 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:57:41,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:41,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:41,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:41,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:42,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:44,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:44,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:45,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:57:45,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:57:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:57:46,729 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:46,730 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:46,730 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:46,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:47,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:48,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:49,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:51,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:51,283 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:51,283 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:51,287 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:51,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:52,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:57:55,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:57:56,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:57:56,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:57:56,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:57:56,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:57:57,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:57:58,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:00,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:00,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:00,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:00,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:01,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:01,242 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:01,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:01,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:01,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:02,461 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:06,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:06,048 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:06,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:06,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:06,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:06,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:08,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:09,578 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:58:10,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:10,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:10,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:10,268 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:10,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:11,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:12,471 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:15,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:15,433 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:15,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:15,438 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:15,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:15,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:15,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:16,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:16,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:19,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:19,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:19,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:19,804 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:20,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:22,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:22,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:23,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:23,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:23,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:23,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:24,511 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:26,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:27,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:28,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:28,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:28,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:28,750 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:29,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:30,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:30,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:30,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:33,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:36,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:36,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:36,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:36,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:36,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:38,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:39,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:39,580 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:58:41,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:41,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:41,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:41,373 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:41,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:42,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:44,373 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:45,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:58:45,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:58:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:58:46,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:46,326 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:46,326 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:46,331 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:46,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:48,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:50,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:50,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:50,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:50,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:50,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:51,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:52,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:54,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:54,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:54,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:54,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:58:55,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:58:55,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:58:56,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:58:59,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:58:59,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:58:59,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:58:59,992 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:00,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:00,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:00,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:01,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:02,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:04,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:04,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:04,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:04,104 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:04,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:06,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:07,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:09,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:09,154 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:09,154 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:09,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:09,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:09,582 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:59:10,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:12,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:13,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:13,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:13,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:13,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:14,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:14,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:15,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:17,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:19,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:19,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:19,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:19,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:19,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:20,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:23,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:24,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:24,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:24,209 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:24,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:24,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:26,575 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:29,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:30,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:30,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:30,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:30,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:30,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:30,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:32,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:34,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:34,786 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:34,786 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:34,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:34,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:35,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:36,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:39,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 14:59:40,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:40,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:40,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:40,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:40,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:40,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:42,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:45,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:45,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 14:59:45,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 14:59:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 14:59:49,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:49,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:49,353 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:49,358 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:49,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 14:59:50,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:50,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:55,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 14:59:55,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 14:59:55,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 14:59:55,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 14:59:55,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 14:59:56,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 14:59:56,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:00,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:00,790 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:00,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:00,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:00,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:00,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:01,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:02,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:05,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:05,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:05,473 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:05,478 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:05,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:06,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:06,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:09,589 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:00:10,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:10,965 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:10,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:10,971 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:11,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:11,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:12,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:15,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:16,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:16,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:16,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:16,568 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:16,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:17,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:18,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:22,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:22,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:22,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:22,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:22,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:23,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:24,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:28,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:28,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:28,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:28,722 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:28,727 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:29,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:30,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:30,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:30,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:30,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:33,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:35,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:35,323 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:35,323 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:35,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:35,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:36,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:39,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:39,588 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:00:39,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:39,936 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:39,936 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:39,941 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:40,646 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:42,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:44,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:45,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:00:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:00:45,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:00:45,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:45,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:45,938 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:45,944 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:46,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:48,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:49,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:51,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:51,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:51,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:51,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:51,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:00:52,660 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:55,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:00:55,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:00:55,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:00:55,784 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:00:55,789 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:00:56,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:00:56,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:00,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:00,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:00,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:00,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:03,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:03,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:03,478 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:03,482 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:03,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:04,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:06,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:08,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:08,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:08,722 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:08,727 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:09,593 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:01:09,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:10,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:11,594 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:15,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:15,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:16,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:17,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:17,197 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:17,197 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:17,202 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:17,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:18,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:22,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:25,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:25,619 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:25,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:25,624 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:25,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:26,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:27,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:30,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:30,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:30,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:31,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:31,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:31,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:31,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:32,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:32,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:32,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:37,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:39,592 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:01:39,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:39,876 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:39,876 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:39,881 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:40,730 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:40,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:42,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:43,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:43,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:43,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:43,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:44,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:44,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:01:45,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:01:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:01:47,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:48,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:48,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:48,262 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:48,267 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:48,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:50,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:53,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:53,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:53,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:53,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:53,739 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:53,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:01:54,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:01:58,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:01:58,349 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:01:58,349 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:01:58,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:01:58,354 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:01:58,749 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:00,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:00,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:00,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:00,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:03,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:03,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:03,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:03,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:03,695 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:03,754 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:04,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:08,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:08,219 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:08,219 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:08,224 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:08,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:09,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:09,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:02:10,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:13,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:13,057 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:13,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:13,062 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:13,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:14,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:15,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:15,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:15,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:18,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:18,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:18,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:18,717 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:18,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:20,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:20,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:23,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:23,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:23,780 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:23,785 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:24,778 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:24,778 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:25,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:28,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:28,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:28,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:28,071 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:28,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:30,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:30,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:30,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:30,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:32,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:32,723 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:32,723 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:32,728 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:32,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:34,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:36,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:37,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:37,808 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:37,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:37,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:37,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:38,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:39,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:02:42,586 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:42,586 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:42,587 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:42,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:42,592 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:42,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:44,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:02:45,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:02:45,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:02:47,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:47,008 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:47,008 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:47,014 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:47,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:48,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:48,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:53,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:02:55,306 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:02:55,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:02:55,307 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:02:55,312 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:02:55,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:02:56,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:02:58,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:00,514 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:00,515 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:00,515 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:00,520 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:00,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:00,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:00,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:02,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:03,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:05,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:05,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:05,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:05,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:05,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:06,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:09,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:09,589 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:09,589 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:09,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:09,594 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:09,599 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:03:09,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:10,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:14,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:15,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:15,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:15,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:15,945 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:15,945 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:15,951 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:16,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:16,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:19,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:20,335 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:20,336 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:20,336 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:20,341 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:20,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:22,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:24,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:24,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:24,500 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:24,505 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:24,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:25,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:26,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:30,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:30,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:30,576 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:30,577 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:30,582 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:30,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:30,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:30,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:32,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:35,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:35,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:35,746 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:35,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:35,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:35,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:36,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:39,604 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:03:40,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:40,468 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:40,468 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:40,473 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:40,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:41,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:42,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:45,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:45,359 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:45,359 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:45,364 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:45,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:03:45,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:03:45,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:03:45,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:46,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:46,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:49,822 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:49,823 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:49,823 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:49,828 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:49,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:50,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:52,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:54,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:54,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:54,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:54,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:54,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:03:56,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:03:58,509 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:03:58,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:03:58,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:03:58,990 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:03:58,996 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:03:59,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:00,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:00,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:00,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:03,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:06,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:06,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:06,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:06,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:06,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:06,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:09,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:09,603 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:04:10,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:10,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:10,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:10,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:10,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:12,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:14,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:15,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:16,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:16,614 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:16,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:16,619 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:16,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:18,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:19,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:21,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:21,283 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:21,284 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:21,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:21,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:22,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:25,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:25,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:25,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:25,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:25,661 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:25,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:26,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:30,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:30,139 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:30,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:30,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:30,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:30,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:30,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:30,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:32,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:35,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:36,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:36,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:36,451 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:36,456 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:36,966 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:38,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:39,628 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:04:40,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:40,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:40,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:40,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:40,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:41,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:42,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:45,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:45,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:45,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:45,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:45,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:04:45,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:04:45,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:04:45,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:46,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:46,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:51,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:51,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:51,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:51,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:51,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:52,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:52,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:04:57,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:04:57,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:04:57,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:04:57,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:04:57,687 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:04:58,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:04:59,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:00,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:00,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:01,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:01,966 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:01,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:01,971 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:02,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:03,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:03,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:07,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:07,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:07,258 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:07,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:08,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:08,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:09,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:09,607 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:05:12,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:12,350 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:12,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:12,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:13,018 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:13,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:15,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:15,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:18,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:18,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:18,240 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:18,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:19,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:19,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:21,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:24,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:25,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:25,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:25,321 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:25,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:26,031 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:27,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:29,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:29,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:29,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:29,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:29,750 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:30,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:30,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:30,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:31,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:34,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:35,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:35,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:35,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:35,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:36,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:37,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:39,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:05:40,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:40,191 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:40,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:40,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:40,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:41,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:43,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:44,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:44,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:44,818 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:44,823 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:45,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:45,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:05:45,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:05:45,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:05:45,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:47,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:50,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:05:51,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:05:51,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:05:51,244 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:05:51,249 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:05:52,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:05:53,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:05:56,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:00,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:00,187 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:00,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:00,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:00,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:00,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:00,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:01,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:01,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:01,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:05,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:05,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:05,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:05,308 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:06,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:07,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:07,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:09,242 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:09,243 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:09,243 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:09,248 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:09,611 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:06:10,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:11,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:12,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:13,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:13,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:13,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:13,102 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:13,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:15,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:15,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:15,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:15,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:17,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:18,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:18,207 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:18,207 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:18,212 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:19,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:21,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:23,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:23,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:23,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:23,047 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:23,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:23,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:25,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:27,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:27,161 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:27,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:27,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:27,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:28,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:29,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:30,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:30,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:32,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:32,661 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:32,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:32,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:33,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:33,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:35,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:38,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:38,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:38,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:38,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:39,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:39,614 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:06:39,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:41,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:44,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:44,161 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:44,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:44,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:44,206 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:45,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:45,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:45,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:06:45,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:06:45,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:06:48,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:48,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:48,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:48,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:49,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:50,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:51,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:53,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:53,702 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:53,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:53,707 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:54,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:06:55,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:06:56,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:06:58,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:06:58,305 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:06:58,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:06:58,310 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:06:59,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:00,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:00,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:01,225 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:01,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:05,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:05,962 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:05,962 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:05,967 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:06,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:07,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:07,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:09,615 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:07:11,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:11,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:11,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:11,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:12,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:12,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:13,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:15,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:16,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:16,391 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:16,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:16,396 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:17,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:18,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:19,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:21,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:21,906 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:21,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:21,911 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:22,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:23,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:25,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:26,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:26,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:26,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:26,640 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:27,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:29,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:29,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:30,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:30,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:31,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:31,570 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:31,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:31,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:32,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:33,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:35,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:36,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:36,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:36,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:36,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:37,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:39,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:39,618 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:07:40,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:40,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:40,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:40,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:40,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:41,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:43,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:45,498 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:45,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:45,499 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:45,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:45,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:07:45,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:07:45,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:07:45,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:46,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:47,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:49,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:49,910 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:49,910 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:49,915 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:50,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:50,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:53,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:07:55,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:07:56,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:07:56,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:07:56,159 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:07:56,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:07:56,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:07:59,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:00,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:01,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:05,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:05,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:05,289 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:05,293 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:06,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:07,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:07,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:09,623 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:08:12,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:13,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:13,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:13,273 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:13,278 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:13,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:15,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:15,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:17,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:17,349 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:17,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:17,355 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:18,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:18,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:19,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:23,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:24,506 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:24,507 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:24,507 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:24,512 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:25,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:27,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:28,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:30,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:30,389 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:30,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:30,395 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:30,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:30,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:30,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:31,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:33,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:33,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:36,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:36,157 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:36,157 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:36,162 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:36,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:39,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:39,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:39,622 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:08:40,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:40,504 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:40,504 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:40,509 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:41,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:43,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:44,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:45,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:08:45,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:08:45,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:08:46,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:46,378 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:46,378 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:46,383 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:47,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:49,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:50,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:54,839 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:54,840 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:54,840 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:54,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:55,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:08:55,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:08:57,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:08:59,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:08:59,335 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:08:59,335 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:08:59,341 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:08:59,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:00,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:00,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:00,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:01,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:03,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:03,827 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:03,827 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:03,832 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:04,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:05,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:06,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:09,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:09:10,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:10,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:10,164 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:10,169 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:10,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:12,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:13,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:15,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:15,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:15,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:17,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:18,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:18,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:18,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:18,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:18,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:21,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:22,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:22,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:22,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:22,639 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:23,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:23,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:25,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:28,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:28,622 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:28,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:28,627 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:29,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:29,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:30,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:30,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:30,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:31,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:32,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:32,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:32,848 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:32,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:33,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:34,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:35,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:37,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:37,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:37,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:37,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:38,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:39,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:39,626 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:09:40,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:42,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:42,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:42,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:42,900 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:43,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:45,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:45,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:09:45,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:09:45,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:09:45,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:47,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:47,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:47,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:47,245 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:47,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:49,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:51,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:52,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:52,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:52,314 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:52,319 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:52,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:55,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:09:56,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:09:57,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:09:57,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:09:57,645 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:09:57,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:09:58,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:09:59,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:00,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:00,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:01,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:02,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:02,447 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:02,447 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:02,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:03,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:05,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:07,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:07,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:07,353 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:07,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:07,358 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:07,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:09,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:09,628 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:10:11,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:11,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:11,975 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:11,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:12,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:12,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:13,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:15,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:15,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:15,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:16,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:16,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:16,278 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:16,284 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:16,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:18,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:19,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:21,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:21,546 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:21,546 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:21,551 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:22,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:23,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:23,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:26,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:26,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:26,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:26,672 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:27,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:28,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:29,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:30,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:30,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:30,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:33,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:35,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:35,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:35,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:35,083 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:35,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:37,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:39,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:39,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:39,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:39,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:39,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:39,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:39,631 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:10:41,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:44,632 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:45,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:45,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:45,198 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:45,203 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:45,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:45,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:10:45,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:10:45,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:10:47,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:49,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:49,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:49,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:49,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:49,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:50,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:51,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:10:54,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:10:54,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:10:54,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:10:54,494 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:10:55,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:10:55,494 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:10:57,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:00,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:00,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:00,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:02,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:02,941 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:02,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:02,946 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:03,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:05,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:05,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:09,633 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:11:11,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:11,483 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:11,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:11,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:11,488 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:12,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:13,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:15,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:15,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:15,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:16,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:16,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:16,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:16,045 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:16,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:17,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:17,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:22,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:23,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:23,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:23,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:23,596 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:24,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:25,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:27,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:29,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:29,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:29,490 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:29,495 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:29,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:30,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:30,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:31,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:32,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:34,149 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:34,150 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:34,150 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:34,155 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:34,511 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:37,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:38,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:38,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:38,023 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:38,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:38,028 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:38,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:39,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:39,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:11:42,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:42,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:42,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:42,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:42,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:43,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:45,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:45,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:11:45,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:11:45,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:11:48,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:48,541 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:48,542 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:48,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:48,546 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:49,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:51,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:53,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:53,403 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:53,403 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:53,410 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:53,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:54,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:11:55,533 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:11:58,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:11:58,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:11:58,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:11:58,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:11:58,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:11:59,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:00,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:00,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:00,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:00,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:04,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:04,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:04,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:04,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:04,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:05,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:07,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:09,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:12:10,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:10,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:10,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:10,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:10,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:11,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:11,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:14,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:14,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:14,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:14,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:15,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:15,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:15,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:15,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:16,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:17,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:18,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:18,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:18,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:18,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:19,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:21,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:22,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:23,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:23,764 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:23,764 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:23,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:24,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:25,582 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:28,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:28,513 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:28,514 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:28,514 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:28,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:28,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:30,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:30,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:31,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:32,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:32,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:32,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:32,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:33,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:33,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:35,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:38,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:38,652 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:38,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:38,657 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:39,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:39,642 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:12:39,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:41,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:43,481 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:43,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:43,482 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:43,487 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:43,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:45,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:45,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:45,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:12:45,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:12:45,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:12:49,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:49,458 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:49,458 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:49,463 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:49,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:50,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:51,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:53,554 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:53,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:53,555 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:53,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:53,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:55,561 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:12:55,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:12:57,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:12:57,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:12:57,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:12:57,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:12:58,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:12:59,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:00,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:00,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:00,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:03,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:03,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:03,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:03,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:03,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:05,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:06,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:08,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:08,204 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:08,204 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:08,209 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:08,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:09,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:09,641 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:13:11,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:12,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:12,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:12,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:12,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:13,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:15,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:15,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:15,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:15,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:16,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:17,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:17,264 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:17,264 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:17,270 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:17,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:19,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:22,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:23,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:23,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:23,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:23,066 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:23,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:25,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:27,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:27,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:27,490 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:27,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:27,495 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:27,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:29,651 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:30,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:30,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:30,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:32,374 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:32,374 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:32,374 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:32,379 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:32,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:33,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:35,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:37,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:37,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:37,831 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:37,836 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:38,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:38,837 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:39,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:13:39,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:44,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:45,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:13:45,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:13:45,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:13:47,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:47,393 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:47,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:47,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:47,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:49,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:50,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:51,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:51,339 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:51,339 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:51,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:51,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:53,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:13:56,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:13:56,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:13:56,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:13:56,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:13:56,677 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:13:56,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:13:59,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:00,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:00,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:01,917 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:02,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:02,989 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:02,989 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:02,994 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:03,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:05,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:07,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:08,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:08,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:08,658 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:08,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:08,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:09,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:14:11,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:12,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:14,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:14,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:14,361 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:14,366 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:14,718 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:15,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:15,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:15,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:17,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:17,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:22,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:22,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:22,532 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:22,537 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:22,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:23,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:25,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:26,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:26,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:26,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:26,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:27,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:28,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:29,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:30,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:30,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:30,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:33,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:34,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:34,381 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:34,381 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:34,386 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:34,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:37,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:39,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:39,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:14:42,839 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:42,840 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:42,840 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:42,845 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:43,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:44,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:45,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:14:45,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:14:45,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:14:45,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:48,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:48,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:48,286 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:48,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:48,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:49,754 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:50,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:52,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:52,959 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:52,959 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:52,964 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:53,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:55,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:14:55,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:14:58,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:14:58,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:14:58,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:14:58,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:14:58,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:14:59,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:00,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:00,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:01,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:02,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:02,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:02,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:02,714 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:02,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:05,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:06,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:06,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:06,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:06,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:06,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:07,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:09,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:15:09,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:12,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:13,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:13,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:13,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:13,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:13,778 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:15,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:15,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:15,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:15,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:17,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:18,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:18,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:18,402 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:18,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:18,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:21,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:23,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:23,074 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:23,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:23,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:23,079 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:23,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:25,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:28,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:29,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:29,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:29,916 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:29,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:30,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:30,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:31,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:33,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:34,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:34,424 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:34,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:34,430 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:34,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:37,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:39,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:39,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:39,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:39,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:39,066 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:39,652 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:15:39,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:41,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:43,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:43,442 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:43,442 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:43,447 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:43,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:44,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:45,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:15:45,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:15:45,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:15:45,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:48,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:48,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:48,276 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:48,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:48,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:49,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:50,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:53,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:53,572 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:53,572 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:53,577 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:53,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:55,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:15:55,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:15:57,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:15:57,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:15:57,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:15:57,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:15:57,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:15:59,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:00,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:00,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:00,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:02,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:02,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:02,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:02,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:02,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:05,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:06,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:07,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:07,554 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:07,554 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:07,581 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:07,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:08,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:09,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:16:12,658 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:12,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:12,710 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:12,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:12,715 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:12,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:14,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:15,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:15,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:17,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:17,537 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:17,537 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:17,543 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:17,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:18,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:18,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:21,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:21,779 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:21,779 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:21,784 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:21,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:22,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:23,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:27,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:27,504 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:27,504 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:27,509 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:27,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:28,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:29,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:30,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:30,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:33,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:33,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:33,035 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:33,040 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:33,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:34,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:35,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:38,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:38,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:38,172 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:38,177 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:38,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:38,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:39,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:16:40,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:43,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:43,946 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:43,946 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:43,951 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:44,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:44,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:45,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:16:45,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:16:45,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:16:45,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:50,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:50,573 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:50,573 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:50,578 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:50,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:51,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:52,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:54,821 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:54,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:54,822 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:54,826 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:54,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:16:56,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:16:56,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:16:59,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:16:59,694 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:16:59,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:16:59,699 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:16:59,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:00,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:00,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:00,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:01,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:05,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:05,811 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:05,812 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:05,817 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:05,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:06,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:07,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:09,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:17:10,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:10,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:10,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:10,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:10,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:12,966 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:13,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:15,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:15,184 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:15,184 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:15,189 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:15,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:15,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:15,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:16,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:18,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:23,493 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:23,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:23,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:23,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:23,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:24,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:24,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:29,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:30,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:30,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:30,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:30,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:30,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:30,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:30,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:30,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:34,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:35,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:35,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:35,473 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:35,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:35,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:36,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:39,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:39,381 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:39,382 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:39,386 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:39,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:17:39,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:40,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:40,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:44,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:44,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:44,859 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:44,864 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:44,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:45,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:17:45,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:17:45,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:17:45,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:47,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:50,306 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:50,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:50,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:50,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:51,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:17:51,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:51,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:56,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:17:56,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:17:56,894 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:17:56,894 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:17:56,899 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:17:57,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:17:59,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:00,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:00,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:01,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:02,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:02,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:02,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:02,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:03,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:05,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:07,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:09,662 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:18:11,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:11,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:11,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:11,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:12,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:13,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:13,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:15,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:15,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:16,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:16,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:16,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:16,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:17,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:17,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:19,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:21,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:21,426 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:21,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:21,431 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:22,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:23,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:24,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:28,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:28,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:28,102 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:28,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:29,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:29,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:30,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:30,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:30,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:30,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:33,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:33,415 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:33,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:33,420 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:34,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:35,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:35,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:39,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:18:40,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:41,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:41,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:41,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:41,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:42,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:43,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:45,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:18:45,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:18:45,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:18:45,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:46,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:46,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:46,899 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:46,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:47,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:49,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:51,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:53,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:53,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:53,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:53,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:18:54,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:18:55,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:18:57,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:18:59,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:18:59,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:18:59,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:18:59,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:00,068 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:00,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:00,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:00,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:01,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:02,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:05,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:05,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:05,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:05,172 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:06,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:07,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:08,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:09,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:19:09,787 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:09,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:09,788 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:09,793 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:10,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:11,081 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:13,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:14,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:14,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:14,744 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:14,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:15,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:15,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:15,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:17,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:18,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:21,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:21,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:21,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:21,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:22,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:23,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:24,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:26,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:26,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:26,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:26,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:27,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:27,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:29,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:30,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:30,602 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:30,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:30,607 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:30,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:30,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:30,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:31,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:31,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:34,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:37,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:37,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:37,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:37,695 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:38,107 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:39,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:39,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:19:40,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:42,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:42,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:42,479 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:42,483 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:43,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:43,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:45,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:19:45,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:19:45,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:19:45,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:50,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:50,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:50,828 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:50,833 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:51,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:51,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:53,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:55,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:55,205 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:55,205 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:55,210 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:19:56,125 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:19:57,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:19:57,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:19:59,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:19:59,650 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:19:59,650 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:19:59,655 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:00,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:00,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:00,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:01,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:02,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:07,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:08,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:08,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:08,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:08,104 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:08,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:09,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:09,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:20:12,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:12,718 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:12,719 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:12,723 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:13,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:13,724 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:15,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:15,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:15,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:15,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:17,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:17,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:17,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:17,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:18,148 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:18,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:19,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:22,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:22,415 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:22,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:22,420 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:23,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:23,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:24,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:28,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:28,421 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:28,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:28,426 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:29,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:29,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:29,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:30,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:30,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:30,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:34,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:36,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:36,579 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:36,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:36,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:37,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:37,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:39,673 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:20:40,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:45,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:45,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:45,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:45,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:45,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:45,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:20:45,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:20:45,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:20:45,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:47,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:49,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:49,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:49,512 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:49,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:50,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:51,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:51,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:54,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:54,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:54,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:54,437 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:55,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:55,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:20:57,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:20:58,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:20:58,636 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:20:58,636 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:20:58,641 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:20:59,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:20:59,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:00,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:02,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:02,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:02,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:02,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:02,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:03,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:05,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:07,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:09,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:21:09,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:09,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:09,987 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:09,992 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:10,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:11,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:12,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:14,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:14,641 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:14,641 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:14,646 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:15,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:15,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:15,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:15,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:18,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:19,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:19,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:19,168 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:19,173 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:19,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:21,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:24,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:24,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:24,276 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:24,276 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:24,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:25,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:25,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:28,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:28,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:28,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:28,925 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:29,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:29,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:30,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:30,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:30,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:31,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:33,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:33,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:33,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:33,649 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:34,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:35,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:35,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:38,759 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:38,760 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:38,760 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:38,765 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:39,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:39,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:39,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:21:41,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:44,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:44,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:44,170 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:44,175 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:44,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:45,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:45,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:21:45,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:21:45,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:21:46,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:50,265 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:50,266 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:50,266 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:50,271 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:51,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:51,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:52,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:54,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:54,479 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:54,480 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:54,484 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:21:55,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:21:55,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:21:57,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:21:59,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:21:59,445 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:21:59,445 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:21:59,450 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:00,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:00,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:00,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:01,253 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:02,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:04,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:04,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:04,343 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:04,348 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:05,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:05,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:08,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:09,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:09,123 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:09,123 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:09,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:09,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:09,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:22:11,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:13,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:14,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:14,585 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:14,585 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:14,590 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:15,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:15,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:15,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:15,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:18,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:21,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:21,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:21,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:21,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:22,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:23,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:24,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:26,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:26,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:26,364 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:26,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:27,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:27,280 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:30,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:30,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:30,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:31,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:31,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:31,209 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:31,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:31,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:33,286 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:36,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:36,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:36,566 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:36,566 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:36,571 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:37,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:37,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:39,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:22:41,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:41,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:41,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:41,014 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:41,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:42,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:43,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:45,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:45,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:45,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:45,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:45,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:22:45,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:22:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:22:46,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:47,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:47,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:51,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:51,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:51,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:51,494 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:52,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:53,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:53,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:22:56,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:22:56,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:22:56,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:22:56,085 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:22:56,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:22:57,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:22:59,086 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:00,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:00,599 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:00,599 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:00,604 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:00,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:00,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:00,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:01,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:01,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:04,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:06,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:06,344 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:06,344 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:06,349 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:07,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:07,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:09,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:23:10,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:10,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:10,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:10,235 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:10,240 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:10,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:11,325 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:14,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:14,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:14,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:14,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:15,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:15,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:15,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:15,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:15,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:15,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:19,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:19,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:19,366 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:19,371 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:20,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:21,354 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:21,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:24,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:24,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:24,345 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:24,350 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:24,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:25,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:27,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:30,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:30,001 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:30,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:30,006 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:30,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:30,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:30,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:31,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:32,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:36,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:36,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:36,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:36,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:37,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:37,372 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:38,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:39,686 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:23:41,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:41,497 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:41,497 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:41,502 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:42,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:43,377 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:44,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:45,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:23:45,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:23:45,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:23:49,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:51,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:51,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:51,003 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:51,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:51,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:53,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:23:55,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:23:57,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:23:57,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:23:57,003 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:23:57,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:23:57,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:23:59,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:00,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:00,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:00,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:00,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:04,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:04,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:04,532 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:04,537 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:05,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:05,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:06,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:09,688 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:24:11,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:11,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:11,286 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:11,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:11,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:12,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:13,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:15,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:15,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:15,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:16,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:16,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:16,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:16,471 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:17,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:17,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:17,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:21,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:21,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:21,275 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:21,280 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:21,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:23,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:23,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:27,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:27,200 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:27,200 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:27,205 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:27,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:29,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:29,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:30,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:30,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:33,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:33,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:33,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:33,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:34,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:34,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:35,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:38,585 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:38,586 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:38,586 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:38,591 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:39,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:39,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:39,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:24:40,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:43,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:43,387 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:43,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:43,392 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:43,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:45,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:45,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:24:45,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:24:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:24:45,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:48,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:48,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:48,831 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:48,836 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:49,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:49,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:51,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:53,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:53,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:53,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:53,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:54,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:24:55,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:57,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:24:58,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:24:58,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:24:58,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:24:58,877 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:24:59,508 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:24:59,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:00,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:00,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:00,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:02,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:04,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:04,012 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:04,012 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:04,017 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:04,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:05,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:08,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:08,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:08,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:08,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:08,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:09,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:09,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:09,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:25:13,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:15,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:15,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:15,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:16,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:16,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:16,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:16,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:17,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:17,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:18,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:23,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:25,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:25,029 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:25,029 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:25,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:25,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:25,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:29,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:29,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:29,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:29,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:29,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:30,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:30,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:30,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:30,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:31,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:34,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:34,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:34,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:34,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:34,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:35,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:35,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:39,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:25:40,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:40,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:40,755 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:40,755 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:40,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:41,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:41,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:45,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:25:45,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:25:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:25:45,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:46,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:46,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:46,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:46,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:46,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:47,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:50,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:50,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:50,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:50,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:50,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:51,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:51,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:55,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:55,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:55,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:55,119 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:25:55,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:25:57,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:25:57,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:25:59,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:25:59,857 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:25:59,857 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:25:59,868 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:00,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:00,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:00,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:00,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:01,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:02,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:05,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:05,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:05,324 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:05,329 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:05,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:07,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:08,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:09,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:26:10,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:10,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:10,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:10,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:11,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:11,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:13,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:15,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:15,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:15,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:18,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:19,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:19,205 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:19,205 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:19,211 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:19,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:21,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:24,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:25,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:25,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:25,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:25,142 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:25,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:25,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:29,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:29,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:29,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:29,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:29,804 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:30,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:30,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:31,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:33,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:33,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:33,863 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:33,868 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:34,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:34,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:35,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:38,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:38,662 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:38,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:38,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:39,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:39,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:39,701 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:26:40,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:42,658 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:42,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:42,659 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:42,664 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:43,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:43,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:45,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:26:45,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:26:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:26:45,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:49,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:49,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:49,514 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:49,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:49,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:51,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:26:51,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:54,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:26:54,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:26:54,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:26:54,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:26:55,668 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:26:55,668 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:26:56,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:00,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:00,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:00,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:01,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:01,307 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:01,307 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:01,312 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:01,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:02,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:03,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:07,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:09,700 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:27:09,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:09,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:09,885 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:09,890 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:10,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:11,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:12,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:13,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:13,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:13,878 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:13,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:14,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:15,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:15,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:15,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:15,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:17,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:18,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:18,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:18,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:18,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:18,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:19,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:23,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:24,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:24,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:24,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:24,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:25,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:25,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:28,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:30,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:30,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:30,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:30,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:30,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:30,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:30,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:31,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:33,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:34,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:34,449 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:34,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:34,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:34,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:35,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:39,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:39,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:39,172 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:39,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:39,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:39,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:27:39,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:39,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:43,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:43,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:43,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:43,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:43,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:44,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:45,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:27:45,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:27:45,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:27:45,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:48,839 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:48,839 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:48,840 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:48,844 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:49,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:49,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:49,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:52,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:52,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:52,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:52,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:53,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:27:53,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:55,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:27:57,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:27:57,942 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:27:57,942 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:27:57,947 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:27:58,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:27:59,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:00,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:00,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:00,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:00,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:03,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:03,280 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:03,280 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:03,285 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:03,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:05,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:06,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:09,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:28:10,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:10,346 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:10,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:10,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:10,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:11,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:11,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:14,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:14,771 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:14,772 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:14,777 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:15,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:15,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:15,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:15,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:15,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:16,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:21,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:23,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:23,029 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:23,029 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:23,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:23,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:23,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:27,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:27,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:27,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:27,092 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:27,097 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:27,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:27,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:30,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:30,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:30,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:31,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:31,603 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:31,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:31,608 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:31,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:32,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:33,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:37,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:37,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:37,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:37,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:37,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:38,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:39,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:28:39,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:42,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:28:45,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:28:45,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:28:46,385 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:46,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:46,386 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:46,391 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:46,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:47,836 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:48,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:51,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:51,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:51,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:51,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:51,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:53,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:28:53,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:56,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:28:56,067 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:28:56,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:28:56,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:28:56,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:28:57,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:28:59,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:00,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:00,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:00,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:01,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:01,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:01,645 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:01,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:01,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:03,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:04,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:07,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:07,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:07,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:07,672 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:07,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:09,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:29:09,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:09,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:13,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:13,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:13,722 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:13,727 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:13,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:14,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:15,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:15,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:15,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:15,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:18,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:18,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:18,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:18,264 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:18,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:19,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:20,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:23,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:23,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:23,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:23,014 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:23,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:23,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:26,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:27,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:27,726 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:27,726 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:27,731 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:27,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:29,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:30,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:30,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:30,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:31,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:34,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:34,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:34,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:34,690 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:34,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:35,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:36,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:39,711 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:29:42,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:43,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:43,132 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:43,132 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:43,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:43,894 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:43,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:29:45,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:29:45,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:29:47,917 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:48,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:48,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:48,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:48,993 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:49,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:49,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:53,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:29:53,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:29:53,624 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:29:53,624 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:29:53,629 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:29:53,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:29:55,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:29:58,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:00,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:00,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:00,019 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:00,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:00,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:00,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:00,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:01,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:03,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:07,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:07,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:07,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:07,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:07,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:09,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:30:09,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:09,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:12,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:12,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:12,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:12,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:12,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:13,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:15,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:15,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:15,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:15,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:16,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:16,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:16,807 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:16,812 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:16,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:17,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:20,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:23,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:23,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:23,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:23,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:23,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:25,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:26,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:27,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:27,450 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:27,450 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:27,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:27,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:29,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:30,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:30,684 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:31,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:33,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:33,363 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:33,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:33,368 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:33,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:33,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:37,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:39,715 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:30:41,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:41,772 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:41,773 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:41,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:41,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:42,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:43,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:45,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:30:45,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:30:45,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:30:46,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:46,475 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:46,475 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:46,480 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:46,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:47,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:48,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:52,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:52,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:52,638 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:52,643 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:52,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:53,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:30:53,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:56,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:30:56,524 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:30:56,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:30:56,529 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:30:56,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:30:57,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:30:59,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:00,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:00,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:01,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:01,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:01,030 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:01,035 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:01,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:01,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:05,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:05,521 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:05,522 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:05,522 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:05,527 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:05,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:07,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:09,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:09,382 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:09,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:09,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:09,718 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:31:10,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:10,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:10,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:14,258 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:14,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:14,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:14,264 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:15,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:15,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:15,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:15,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:15,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:16,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:20,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:20,605 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:20,605 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:20,611 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:21,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:21,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:22,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:26,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:26,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:26,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:26,105 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:27,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:27,105 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:28,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:30,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:30,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:31,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:31,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:31,888 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:31,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:32,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:32,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:34,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:36,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:36,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:36,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:36,943 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:37,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:37,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:38,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:39,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:31:41,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:41,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:41,746 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:41,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:42,068 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:43,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:44,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:45,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:31:45,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:31:45,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:31:46,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:46,191 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:46,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:46,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:47,073 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:48,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:49,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:50,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:50,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:50,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:50,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:51,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:52,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:54,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:31:55,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:31:55,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:31:55,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:31:55,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:31:56,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:31:58,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:31:59,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:00,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:00,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:00,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:00,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:00,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:00,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:01,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:02,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:04,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:08,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:08,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:08,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:08,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:09,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:09,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:32:10,125 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:10,724 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:15,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:15,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:15,180 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:15,185 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:15,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:15,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:15,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:15,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:16,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:16,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:19,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:19,386 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:19,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:19,392 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:20,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:20,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:21,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:23,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:23,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:23,929 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:23,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:24,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:26,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:26,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:28,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:28,605 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:28,605 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:28,611 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:29,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:30,148 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:30,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:30,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:30,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:31,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:33,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:33,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:33,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:33,125 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:33,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:34,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:37,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:37,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:37,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:37,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:37,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:38,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:38,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:39,725 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:32:42,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:42,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:42,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:42,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:42,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:43,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:44,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:45,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:32:45,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:32:45,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:32:46,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:46,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:46,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:46,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:47,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:47,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:48,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:52,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:54,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:54,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:54,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:54,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:32:55,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:32:56,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:32:57,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:32:59,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:32:59,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:32:59,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:32:59,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:00,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:00,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:00,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:00,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:00,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:03,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:03,746 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:03,746 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:03,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:03,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:04,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:06,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:08,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:08,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:08,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:08,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:08,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:09,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:09,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:33:10,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:14,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:14,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:14,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:14,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:14,114 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:14,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:15,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:15,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:16,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:19,481 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:19,481 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:19,482 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:19,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:19,487 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:20,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:20,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:24,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:26,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:26,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:26,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:26,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:27,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:28,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:30,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:30,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:30,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:30,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:30,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:30,961 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:30,961 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:30,967 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:31,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:32,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:35,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:35,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:35,179 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:35,184 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:35,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:36,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:36,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:39,732 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:33:41,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:41,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:41,962 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:41,962 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:41,967 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:42,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:44,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:45,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:33:45,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:33:45,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:33:46,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:46,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:46,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:46,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:47,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:47,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:48,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:52,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:33:54,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:33:54,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:33:54,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:33:54,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:33:55,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:33:56,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:33:57,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:00,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:00,629 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:00,629 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:00,634 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:00,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:00,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:01,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:02,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:02,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:07,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:07,517 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:07,517 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:07,523 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:08,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:08,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:08,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:09,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:34:13,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:13,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:13,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:13,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:13,908 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:14,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:15,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:15,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:16,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:18,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:19,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:19,186 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:19,186 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:19,191 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:19,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:20,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:24,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:24,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:24,621 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:24,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:24,627 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:25,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:26,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:28,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:28,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:28,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:28,776 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:29,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:29,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:30,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:30,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:30,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:34,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:34,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:34,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:34,280 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:34,280 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:35,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:36,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:39,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:34:40,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:40,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:40,179 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:40,184 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:40,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:41,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:42,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:45,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:34:45,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:34:45,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:34:46,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:48,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:48,721 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:48,722 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:48,727 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:49,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:50,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:52,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:53,270 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:53,271 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:53,271 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:53,276 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:34:53,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:34:54,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:34:58,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:34:59,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:34:59,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:34:59,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:34:59,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:00,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:00,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:00,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:03,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:05,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:05,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:05,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:05,037 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:05,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:06,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:09,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:09,522 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:09,523 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:09,523 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:09,528 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:09,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:35:10,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:10,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:13,491 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:13,491 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:13,492 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:13,497 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:14,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:14,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:14,497 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:15,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:15,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:18,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:18,312 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:18,312 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:18,317 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:18,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:20,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:20,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:22,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:22,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:22,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:22,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:23,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:24,332 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:25,801 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:27,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:27,421 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:27,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:27,426 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:28,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:28,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:30,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:30,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:30,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:32,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:32,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:32,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:32,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:32,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:34,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:36,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:36,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:36,791 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:36,792 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:36,796 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:37,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:38,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:39,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:35:41,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:43,142 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:43,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:43,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:43,148 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:43,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:44,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:45,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:35:45,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:35:45,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:35:46,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:48,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:48,708 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:48,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:48,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:49,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:50,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:52,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:53,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:53,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:53,178 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:53,183 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:53,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:35:54,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:35:58,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:35:59,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:35:59,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:35:59,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:35:59,179 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:35:59,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:00,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:00,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:00,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:00,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:03,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:03,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:03,532 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:03,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:03,537 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:04,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:04,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:08,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:09,739 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:36:09,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:09,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:09,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:09,870 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:10,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:12,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:13,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:14,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:14,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:14,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:14,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:15,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:15,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:15,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:16,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:18,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:19,514 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:19,515 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:19,515 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:19,520 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:20,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:20,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:24,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:27,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:27,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:27,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:27,659 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:28,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:28,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:29,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:30,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:30,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:30,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:33,465 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:33,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:33,466 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:33,471 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:34,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:34,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:35,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:37,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:37,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:37,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:37,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:38,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:38,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:39,741 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:36:40,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:42,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:42,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:42,165 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:42,170 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:42,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:44,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:45,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:36:45,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:36:45,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:36:45,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:47,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:47,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:47,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:47,045 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:47,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:48,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:51,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:52,139 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:52,140 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:52,140 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:52,145 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:52,423 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:36:54,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:56,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:36:57,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:36:57,426 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:36:57,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:36:57,431 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:36:58,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:36:58,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:00,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:00,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:00,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:01,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:05,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:05,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:05,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:05,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:06,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:06,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:07,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:09,746 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:37:11,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:11,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:11,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:11,284 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:11,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:12,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:13,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:15,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:15,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:16,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:16,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:16,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:16,018 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:16,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:18,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:19,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:20,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:20,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:20,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:20,437 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:20,452 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:22,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:24,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:25,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:25,801 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:25,801 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:25,806 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:26,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:26,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:29,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:30,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:30,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:30,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:31,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:31,259 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:31,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:31,264 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:31,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:32,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:35,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:36,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:36,789 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:36,789 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:36,794 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:37,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:38,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:39,746 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:37:40,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:41,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:41,171 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:41,171 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:41,176 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:41,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:42,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:45,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:45,465 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:45,465 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:45,470 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:45,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:45,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:37:45,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:37:45,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:37:45,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:46,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:49,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:49,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:49,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:49,974 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:50,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:51,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:52,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:37:56,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:37:56,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:37:56,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:37:56,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:37:56,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:37:57,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:37:58,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:00,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:00,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:00,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:01,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:03,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:03,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:03,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:03,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:04,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:04,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:07,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:07,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:07,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:07,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:07,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:08,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:08,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:09,751 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:38:12,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:12,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:12,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:12,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:12,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:13,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:14,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:15,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:15,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:17,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:17,471 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:17,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:17,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:17,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:18,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:18,551 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:22,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:22,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:22,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:22,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:23,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:23,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:24,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:28,294 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:28,295 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:28,295 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:28,301 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:28,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:29,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:30,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:30,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:30,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:30,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:32,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:32,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:32,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:32,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:33,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:34,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:34,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:37,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:37,187 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:37,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:37,193 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:37,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:38,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:39,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:38:40,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:45,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:45,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:45,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:45,561 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:45,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:45,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:38:45,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:38:45,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:38:45,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:46,579 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:50,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:38:52,118 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:38:52,119 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:38:52,119 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:38:52,123 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:38:52,592 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:38:54,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:38:56,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:00,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:00,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:00,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:01,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:01,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:01,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:01,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:01,509 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:01,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:02,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:05,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:05,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:05,938 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:05,943 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:06,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:06,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:06,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:09,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:39:10,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:10,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:10,463 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:10,468 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:10,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:12,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:12,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:14,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:14,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:14,334 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:14,339 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:14,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:15,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:15,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:15,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:16,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:17,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:22,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:22,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:22,916 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:22,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:22,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:23,622 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:24,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:27,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:29,090 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:29,093 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:29,093 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:29,099 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:29,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:30,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:30,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:30,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:30,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:32,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:33,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:33,528 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:33,529 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:33,534 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:33,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:34,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:38,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:38,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:38,854 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:38,855 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:38,860 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:39,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:39,755 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:39:40,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:43,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:45,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:39:45,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:39:45,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:39:47,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:47,411 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:47,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:47,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:47,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:48,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:49,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:52,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:52,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:52,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:52,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:52,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:54,493 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:39:54,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:39:58,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:39:58,704 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:39:58,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:39:58,709 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:39:59,656 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:39:59,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:00,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:00,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:00,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:00,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:03,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:03,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:03,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:03,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:03,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:04,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:05,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:08,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:08,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:08,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:08,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:08,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:09,757 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:40:10,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:10,758 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:14,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:14,991 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:14,991 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:14,996 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:15,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:15,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:15,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:15,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:16,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:20,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:20,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:20,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:20,480 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:20,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:21,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:22,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:24,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:24,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:24,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:24,927 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:25,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:26,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:26,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:29,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:29,278 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:29,278 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:29,283 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:29,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:30,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:30,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:30,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:30,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:31,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:36,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:37,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:37,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:37,581 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:37,586 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:37,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:38,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:39,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:40:41,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:41,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:41,868 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:41,873 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:42,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:42,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:42,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:45,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:40:45,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:40:45,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:40:46,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:46,937 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:46,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:46,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:47,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:47,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:48,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:51,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:51,271 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:51,272 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:51,276 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:51,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:52,754 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:53,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:40:55,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:40:55,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:40:55,899 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:40:55,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:40:56,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:40:56,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:40:58,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:00,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:00,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:00,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:01,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:01,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:01,337 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:01,342 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:01,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:02,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:04,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:05,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:05,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:05,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:05,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:06,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:06,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:09,761 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:41:09,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:11,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:11,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:11,019 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:11,023 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:11,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:12,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:15,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:15,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:15,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:15,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:15,946 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:15,946 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:15,951 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:16,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:16,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:20,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:22,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:22,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:22,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:22,074 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:22,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:22,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:26,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:28,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:28,206 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:28,206 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:28,211 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:28,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:30,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:30,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:30,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:30,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:31,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:33,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:33,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:33,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:33,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:33,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:34,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:37,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:37,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:37,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:37,627 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:37,632 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:37,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:38,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:39,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:41:41,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:41,849 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:41,849 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:41,854 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:41,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:42,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:42,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:45,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:41:45,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:41:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:41:47,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:47,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:47,786 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:47,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:47,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:48,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:48,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:53,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:54,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:54,552 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:54,553 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:54,558 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:54,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:41:56,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:41:59,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:41:59,124 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:41:59,124 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:41:59,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:41:59,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:41:59,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:00,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:00,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:00,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:00,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:04,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:06,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:06,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:06,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:06,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:06,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:08,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:09,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:42:10,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:13,083 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:13,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:13,084 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:13,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:13,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:14,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:15,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:15,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:15,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:18,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:18,191 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:18,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:18,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:18,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:18,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:21,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:24,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:24,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:24,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:24,780 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:24,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:26,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:26,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:30,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:30,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:30,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:30,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:30,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:30,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:30,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:30,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:30,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:31,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:36,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:36,665 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:36,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:36,671 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:36,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:37,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:38,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:39,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:42:42,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:43,491 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:43,492 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:43,492 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:43,497 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:43,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:44,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:45,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:42:45,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:42:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:42:47,917 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:51,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:51,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:51,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:51,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:52,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:42:52,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:53,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:56,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:42:56,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:42:56,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:42:56,487 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:42:56,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:42:58,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:42:58,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:00,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:00,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:00,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:02,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:02,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:02,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:02,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:02,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:02,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:04,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:07,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:07,377 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:07,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:07,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:07,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:08,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:09,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:09,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:43:13,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:13,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:13,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:13,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:13,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:14,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:15,047 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:15,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:15,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:18,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:18,576 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:18,576 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:18,581 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:18,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:20,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:20,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:22,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:22,689 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:22,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:22,694 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:23,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:25,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:25,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:26,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:26,941 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:26,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:26,948 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:27,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:29,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:30,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:30,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:30,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:30,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:32,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:32,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:32,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:32,219 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:33,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:33,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:36,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:37,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:37,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:37,004 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:37,009 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:37,015 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:39,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:39,772 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:43:41,773 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:43,118 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:43,119 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:43,119 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:43,124 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:44,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:45,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:45,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:43:45,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:43:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:43:46,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:51,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:52,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:52,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:52,290 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:52,294 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:53,031 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:43:53,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:57,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:43:57,148 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:43:57,149 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:43:57,149 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:43:57,154 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:43:58,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:43:59,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:00,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:00,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:02,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:02,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:02,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:02,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:02,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:03,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:05,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:06,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:06,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:06,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:06,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:07,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:07,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:09,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:09,777 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:44:11,506 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:11,507 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:11,507 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:11,511 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:12,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:13,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:13,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:15,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:15,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:15,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:18,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:19,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:19,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:19,872 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:19,877 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:20,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:21,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:24,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:24,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:24,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:24,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:24,657 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:25,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:27,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:29,086 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:29,087 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:29,087 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:29,092 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:30,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:30,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:30,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:30,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:30,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:31,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:34,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:34,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:34,370 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:34,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:35,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:35,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:35,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:39,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:39,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:39,591 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:39,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:39,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:44:40,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:40,777 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:41,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:44,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:44,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:44,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:44,775 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:45,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:44:45,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:44:45,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:44:45,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:47,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:49,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:49,283 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:49,283 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:49,288 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:50,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:51,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:44:51,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:55,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:44:55,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:44:55,366 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:44:55,371 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:44:56,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:44:56,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:44:57,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:00,021 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:00,022 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:00,022 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:00,027 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:00,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:00,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:00,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:01,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:01,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:06,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:09,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:09,170 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:09,171 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:09,175 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:09,781 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:45:10,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:11,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:12,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:14,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:14,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:14,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:14,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:15,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:15,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:15,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:15,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:17,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:18,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:18,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:18,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:18,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:19,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:21,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:23,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:23,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:23,286 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:23,286 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:23,291 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:24,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:25,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:27,826 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:27,827 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:27,827 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:27,832 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:28,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:28,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:29,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:30,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:30,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:30,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:32,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:32,516 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:32,516 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:32,521 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:33,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:34,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:35,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:38,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:38,553 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:38,553 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:38,558 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:39,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:39,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:45:39,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:41,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:43,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:43,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:43,324 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:43,329 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:44,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:45,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:45,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:45:45,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:45:45,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:45:48,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:48,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:48,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:48,136 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:48,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:49,199 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:51,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:45:53,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:45:53,856 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:45:53,857 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:45:53,861 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:45:54,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:45:55,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:45:56,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:00,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:00,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:00,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:00,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:00,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:00,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:00,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:01,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:01,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:06,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:08,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:08,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:08,610 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:08,615 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:09,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:09,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:46:11,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:12,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:13,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:13,007 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:13,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:13,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:13,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:15,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:15,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:15,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:17,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:17,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:17,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:17,672 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:18,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:18,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:19,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:21,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:21,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:21,929 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:21,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:22,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:23,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:23,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:26,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:26,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:26,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:26,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:26,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:27,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:29,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:30,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:30,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:30,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:31,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:31,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:31,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:31,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:31,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:33,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:34,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:35,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:35,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:35,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:35,451 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:36,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:37,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:39,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:39,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:46:40,226 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:40,227 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:40,227 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:40,232 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:40,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:41,291 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:44,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:44,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:44,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:44,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:44,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:45,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:46:45,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:46:45,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:46:47,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:49,527 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:49,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:49,528 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:49,532 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:50,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:50,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:46:51,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:54,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:46:54,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:46:54,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:46:54,354 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:46:55,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:46:55,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:46:56,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:00,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:00,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:00,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:00,225 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:00,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:00,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:00,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:00,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:01,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:01,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:04,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:04,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:04,609 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:04,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:05,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:07,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:07,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:09,787 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:47:10,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:10,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:10,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:10,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:11,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:12,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:13,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:15,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:15,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:15,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:16,297 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:16,298 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:16,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:16,302 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:16,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:17,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:18,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:22,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:22,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:22,185 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:22,190 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:22,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:23,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:24,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:26,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:26,431 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:26,432 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:26,436 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:27,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:27,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:29,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:30,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:30,453 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:30,453 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:30,458 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:30,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:30,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:30,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:31,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:31,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:34,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:35,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:35,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:35,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:35,695 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:36,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:37,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:39,789 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:47:40,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:40,191 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:40,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:40,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:40,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:40,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:41,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:45,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:45,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:45,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:45,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:45,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:45,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:47:45,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:47:45,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:47:46,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:47,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:50,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:51,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:51,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:51,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:51,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:52,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:53,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:47:56,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:47:58,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:47:58,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:47:58,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:47:58,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:47:58,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:47:59,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:00,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:00,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:01,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:03,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:03,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:03,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:03,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:04,438 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:05,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:07,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:07,842 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:07,843 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:07,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:07,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:08,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:09,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:09,791 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:48:12,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:13,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:13,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:13,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:13,622 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:14,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:15,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:15,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:15,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:15,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:17,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:19,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:19,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:19,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:19,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:19,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:21,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:23,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:24,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:24,302 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:24,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:24,308 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:24,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:25,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:29,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:30,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:30,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:30,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:31,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:31,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:31,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:31,429 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:31,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:33,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:34,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:35,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:35,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:35,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:35,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:36,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:37,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:39,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:39,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:48:39,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:39,858 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:39,858 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:39,863 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:40,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:41,494 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:44,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:44,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:44,463 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:44,468 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:44,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:45,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:45,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:45,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:48:45,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:48:45,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:48:50,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:52,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:52,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:52,744 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:52,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:53,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:55,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:48:56,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:48:57,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:48:57,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:48:57,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:48:57,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:48:58,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:48:59,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:00,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:00,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:01,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:02,239 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:02,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:02,240 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:02,245 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:02,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:03,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:07,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:09,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:49:10,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:10,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:10,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:11,003 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:11,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:13,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:13,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:15,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:15,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:15,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:18,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:19,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:19,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:19,639 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:19,643 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:20,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:21,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:24,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:24,585 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:24,585 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:24,585 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:24,590 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:25,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:25,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:28,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:28,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:28,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:28,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:29,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:29,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:30,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:30,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:30,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:31,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:34,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:34,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:34,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:34,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:34,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:35,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:35,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:39,797 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:49:40,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:42,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:42,680 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:42,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:42,685 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:43,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:43,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:45,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:49:45,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:49:45,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:49:45,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:47,559 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:47,559 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:47,559 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:47,564 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:47,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:49,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:51,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:53,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:53,162 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:53,162 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:53,167 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:53,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:55,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:49:57,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:49:57,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:49:57,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:49:57,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:49:57,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:49:58,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:49:59,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:00,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:00,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:00,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:02,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:03,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:03,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:03,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:03,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:04,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:05,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:08,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:09,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:09,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:09,351 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:09,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:09,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:09,799 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:50:11,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:13,563 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:13,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:13,564 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:13,569 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:13,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:14,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:15,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:15,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:15,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:15,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:18,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:18,926 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:18,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:18,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:19,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:19,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:21,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:23,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:23,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:23,314 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:23,319 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:23,660 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:25,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:25,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:28,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:28,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:28,126 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:28,131 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:28,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:29,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:30,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:30,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:30,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:30,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:35,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:35,471 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:35,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:35,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:35,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:36,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:37,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:39,801 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:50:40,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:40,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:40,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:40,041 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:40,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:41,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:42,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:45,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:50:45,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:50:45,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:50:47,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:47,431 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:47,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:47,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:47,436 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:47,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:49,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:52,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:52,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:52,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:52,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:52,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:52,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:53,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:50:57,497 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:50:57,498 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:50:57,499 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:50:57,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:50:57,503 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:50:57,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:50:59,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:00,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:00,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:00,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:02,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:05,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:05,983 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:05,983 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:05,988 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:06,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:07,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:08,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:09,806 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:51:10,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:10,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:10,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:10,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:10,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:11,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:13,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:14,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:14,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:14,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:14,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:15,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:15,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:15,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:15,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:17,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:18,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:21,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:21,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:21,324 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:21,330 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:21,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:23,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:24,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:25,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:25,742 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:25,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:25,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:26,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:27,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:29,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:30,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:30,676 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:30,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:30,681 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:30,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:30,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:30,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:30,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:31,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:34,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:35,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:35,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:35,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:35,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:35,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:37,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:39,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:51:40,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:40,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:40,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:40,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:40,682 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:40,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:41,806 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:45,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:45,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:45,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:45,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:45,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:51:45,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:51:45,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:51:45,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:45,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:47,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:49,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:49,568 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:49,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:49,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:49,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:51,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:51,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:54,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:54,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:54,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:51:54,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:51:55,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:51:56,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:51:57,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:51:59,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:51:59,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:51:59,999 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:00,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:00,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:00,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:00,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:00,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:01,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:02,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:05,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:05,534 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:05,534 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:05,539 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:05,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:07,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:08,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:09,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:52:12,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:12,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:12,789 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:12,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:12,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:13,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:13,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:15,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:15,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:15,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:17,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:17,497 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:17,497 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:17,502 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:17,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:19,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:19,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:22,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:22,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:22,218 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:22,223 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:22,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:23,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:25,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:26,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:26,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:26,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:26,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:26,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:27,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:30,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:30,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:30,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:30,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:31,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:31,614 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:31,614 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:31,620 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:31,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:33,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:36,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:36,044 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:36,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:36,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:36,050 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:36,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:37,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:39,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:52:41,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:41,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:41,035 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:41,040 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:41,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:42,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:43,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:45,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:45,332 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:45,332 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:45,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:45,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:52:45,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:52:45,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:52:45,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:47,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:47,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:50,235 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:50,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:50,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:50,240 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:50,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:51,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:53,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:54,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:54,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:54,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:54,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:54,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:52:55,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:52:58,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:52:59,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:52:59,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:52:59,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:52:59,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:52:59,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:00,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:00,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:00,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:01,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:03,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:03,951 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:03,951 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:03,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:03,956 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:04,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:05,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:08,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:09,815 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:53:11,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:11,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:11,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:11,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:11,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:13,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:14,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:15,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:15,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:15,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:17,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:17,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:17,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:17,374 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:17,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:19,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:20,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:22,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:22,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:22,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:22,405 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:22,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:23,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:26,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:27,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:27,132 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:27,132 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:27,138 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:27,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:29,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:30,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:30,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:30,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:31,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:32,315 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:32,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:32,316 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:32,321 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:32,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:33,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:36,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:36,715 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:36,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:36,720 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:36,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:37,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:37,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:39,814 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:53:42,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:45,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:45,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:45,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:45,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:45,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:53:45,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:53:45,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:53:45,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:47,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:47,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:51,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:51,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:51,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:51,966 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:51,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:53,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:53:53,983 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:53:57,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:53:57,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:53:57,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:53:57,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:53:57,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:53:58,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:00,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:00,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:00,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:00,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:03,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:06,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:06,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:06,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:06,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:07,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:08,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:09,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:09,816 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:54:10,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:10,717 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:10,717 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:10,722 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:11,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:12,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:14,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:15,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:15,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:15,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:16,298 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:16,298 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:16,299 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:16,303 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:17,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:18,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:20,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:24,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:24,162 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:24,162 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:24,167 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:25,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:26,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:26,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:29,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:29,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:29,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:29,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:30,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:30,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:30,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:31,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:32,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:36,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:36,708 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:36,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:36,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:37,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:37,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:38,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:39,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:54:40,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:40,742 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:40,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:40,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:41,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:42,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:43,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:45,486 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:45,487 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:45,487 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:45,492 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:45,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:54:45,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:54:45,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:54:46,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:48,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:48,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:50,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:50,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:50,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:50,750 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:51,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:52,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:54:54,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:54:55,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:54:55,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:54:55,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:54:55,268 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:54:56,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:54:58,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:00,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:00,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:00,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:00,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:00,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:00,916 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:00,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:01,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:02,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:05,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:06,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:06,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:06,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:06,074 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:06,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:08,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:09,820 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:55:11,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:11,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:11,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:11,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:11,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:12,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:14,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:15,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:15,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:15,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:16,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:18,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:18,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:18,290 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:18,296 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:19,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:20,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:22,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:23,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:23,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:23,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:23,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:24,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:26,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:27,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:27,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:27,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:27,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:27,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:28,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:30,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:30,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:30,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:32,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:32,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:32,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:32,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:33,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:33,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:34,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:37,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:37,113 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:37,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:37,118 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:37,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:38,193 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:39,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:39,825 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:55:44,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:45,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:45,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:45,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:45,228 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:45,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:55:45,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:55:45,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:55:46,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:48,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:49,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:50,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:50,234 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:50,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:50,239 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:51,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:52,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:55:55,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:55:55,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:55:55,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:55:55,627 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:55:55,632 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:55:56,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:55:58,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:00,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:00,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:00,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:00,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:00,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:00,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:00,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:00,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:01,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:02,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:05,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:07,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:07,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:07,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:07,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:08,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:09,824 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:56:10,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:11,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:11,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:11,349 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:11,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:11,354 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:12,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:14,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:15,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:15,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:15,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:16,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:18,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:18,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:18,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:18,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:18,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:20,264 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:22,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:23,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:23,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:23,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:23,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:24,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:26,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:27,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:28,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:28,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:28,784 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:28,788 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:29,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:30,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:30,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:30,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:32,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:33,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:33,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:33,351 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:33,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:34,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:36,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:38,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:39,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:56:41,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:41,956 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:41,956 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:41,961 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:42,291 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:43,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:44,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:45,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:56:45,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:56:45,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:56:46,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:46,835 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:46,835 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:46,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:47,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:48,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:49,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:51,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:51,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:51,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:51,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:51,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:52,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:56:55,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:56:56,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:56:56,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:56:56,416 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:56:56,422 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:56:57,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:56:58,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:00,423 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:00,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:00,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:00,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:01,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:01,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:01,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:01,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:02,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:04,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:05,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:06,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:06,108 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:06,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:06,113 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:06,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:08,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:09,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:57:10,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:10,689 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:10,689 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:10,694 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:11,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:11,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:12,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:15,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:15,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:15,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:16,534 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:16,535 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:16,535 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:16,540 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:17,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:17,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:18,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:22,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:24,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:24,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:24,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:24,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:25,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:26,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:27,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:30,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:30,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:30,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:30,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:30,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:30,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:30,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:32,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:32,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:34,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:34,625 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:34,625 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:34,629 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:35,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:36,350 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:38,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:39,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:57:40,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:40,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:40,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:40,070 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:40,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:42,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:44,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:45,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:57:45,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:57:45,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:57:45,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:46,004 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:46,004 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:46,009 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:46,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:48,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:50,010 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:50,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:50,732 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:50,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:50,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:51,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:52,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:54,493 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:54,494 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:54,495 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:54,500 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:57:55,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:57:55,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:57:56,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:57:59,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:57:59,618 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:57:59,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:57:59,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:00,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:00,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:00,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:00,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:00,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:02,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:05,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:06,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:06,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:06,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:06,370 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:06,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:08,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:09,833 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:58:11,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:11,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:11,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:11,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:11,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:12,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:14,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:15,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:15,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:15,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:16,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:16,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:16,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:16,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:16,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:17,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:18,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:22,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:22,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:22,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:22,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:22,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:22,396 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:24,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:27,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:27,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:27,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:27,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:27,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:28,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:30,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:30,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:30,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:30,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:32,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:32,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:32,031 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:32,036 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:32,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:33,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:34,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:38,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:38,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:38,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:38,338 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:38,343 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:38,413 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:39,835 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:58:40,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:42,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:42,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:42,958 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:42,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:43,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:43,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:44,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:45,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:58:45,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:58:45,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:58:49,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:51,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:51,553 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:51,554 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:51,559 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:52,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:54,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:58:55,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:58:56,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:58:56,171 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:58:56,172 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:58:56,177 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:58:56,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:58:58,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:00,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:00,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:00,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:00,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:01,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:01,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:01,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:01,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:02,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:04,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:06,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:06,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:06,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:06,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:06,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:06,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:08,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:09,840 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:59:11,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:11,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:11,542 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:11,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:11,547 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:12,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:14,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:15,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:15,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:15,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:16,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:17,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:17,089 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:17,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:17,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:17,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:18,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:22,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:25,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:25,530 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:25,531 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:25,536 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:26,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:27,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:28,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:30,154 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:30,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:30,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:30,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:30,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:30,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:30,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:32,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:32,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:34,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:34,621 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:34,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:34,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:35,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:35,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:38,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:39,839 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 15:59:40,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:40,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:40,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:40,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:41,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:41,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:43,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:45,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:45,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:45,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:45,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:45,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:45,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:45,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 15:59:45,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 15:59:45,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 15:59:48,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:53,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 15:59:53,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 15:59:53,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 15:59:53,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 15:59:54,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 15:59:54,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 15:59:56,497 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 15:59:59,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:00,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:00,460 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:00,460 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:00,465 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:00,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:00,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:00,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:00,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:02,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:04,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:06,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:06,068 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:06,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:06,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:06,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:07,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:09,844 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:00:10,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:11,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:11,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:11,346 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:11,351 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:11,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:13,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:15,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:15,487 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:15,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:15,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:15,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:15,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:15,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:15,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:15,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:17,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:20,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:20,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:20,289 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:20,294 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:20,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:21,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:21,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:25,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:25,852 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:25,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:25,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:26,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:26,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:27,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:29,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:30,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:30,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:30,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:30,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:30,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:30,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:31,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:31,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:35,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:35,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:35,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:35,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:35,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:37,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:37,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:39,844 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:00:40,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:40,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:40,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:40,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:41,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:41,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:42,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:45,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:00:45,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:00:45,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:00:47,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:47,048 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:47,048 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:47,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:47,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:48,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:49,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:51,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:51,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:51,958 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:51,963 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:52,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:53,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:00:53,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:58,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:00:58,425 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:00:58,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:00:58,431 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:00:58,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:00:59,431 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:00:59,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:00,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:00,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:00,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:03,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:03,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:03,478 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:03,483 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:03,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:04,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:05,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:08,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:08,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:08,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:08,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:08,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:09,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:09,845 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:01:09,847 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:13,581 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:13,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:13,582 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:13,587 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:13,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:15,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:15,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:15,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:15,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:15,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:18,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:18,363 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:18,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:18,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:18,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:19,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:21,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:23,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:23,358 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:23,359 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:23,364 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:23,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:25,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:27,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:27,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:27,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:27,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:27,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:28,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:29,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:30,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:30,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:30,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:32,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:34,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:34,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:34,842 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:34,847 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:35,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:35,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:38,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:39,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:39,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:39,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:39,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:39,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:01:40,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:41,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:44,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:44,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:44,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:44,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:44,280 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:44,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:45,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:45,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:01:45,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:01:45,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:01:48,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:48,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:48,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:48,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:49,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:49,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:49,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:53,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:53,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:53,015 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:53,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:53,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:53,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:01:55,021 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:01:58,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:01:58,939 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:01:58,940 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:01:58,945 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:01:59,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:01:59,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:00,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:00,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:00,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:00,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:03,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:03,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:03,361 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:03,366 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:03,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:05,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:06,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:07,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:07,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:07,175 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:07,180 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:07,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:09,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:09,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:02:11,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:11,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:11,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:11,284 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:11,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:12,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:13,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:15,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:15,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:15,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:16,558 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:16,558 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:16,559 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:16,564 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:16,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:17,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:17,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:22,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:22,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:22,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:22,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:22,870 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:23,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:23,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:27,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:28,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:28,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:28,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:28,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:28,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:29,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:30,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:30,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:32,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:33,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:33,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:33,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:33,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:33,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:35,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:37,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:37,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:37,640 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:37,646 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:37,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:38,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:39,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:39,852 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:02:42,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:42,119 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:42,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:42,125 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:42,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:43,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:44,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:45,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:02:45,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:02:45,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:02:46,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:46,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:46,155 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:46,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:46,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:47,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:49,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:51,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:51,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:51,015 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:51,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:51,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:02:51,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:55,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:02:56,000 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:02:56,001 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:02:56,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:02:56,006 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:02:56,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:02:57,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:00,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:00,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:00,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:00,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:00,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:00,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:00,984 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:00,989 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:01,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:01,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:06,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:06,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:06,993 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:06,993 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:06,998 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:07,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:07,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:09,854 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:03:11,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:11,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:11,899 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:11,900 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:11,905 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:12,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:13,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:15,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:15,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:15,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:16,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:16,924 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:16,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:16,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:16,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:17,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:17,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:21,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:22,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:22,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:22,268 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:22,273 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:22,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:23,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:27,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:27,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:27,626 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:27,626 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:27,631 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:27,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:29,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:30,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:30,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:31,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:31,921 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:31,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:31,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:32,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:32,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:33,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:37,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:38,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:38,073 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:38,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:38,079 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:38,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:39,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:39,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:03:42,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:42,332 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:42,333 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:42,338 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:42,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:43,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:43,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:45,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:03:45,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:03:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:03:47,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:47,662 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:47,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:47,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:47,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:48,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:49,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:51,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:51,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:51,864 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:51,870 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:52,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:53,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:53,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:03:56,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:03:56,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:03:56,102 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:03:56,108 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:03:56,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:03:57,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:03:59,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:00,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:00,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:00,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:00,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:00,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:00,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:00,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:01,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:01,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:04,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:07,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:07,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:07,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:07,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:07,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:09,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:09,858 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:04:10,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:12,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:12,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:12,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:12,092 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:12,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:13,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:15,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:15,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:15,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:15,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:16,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:16,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:16,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:16,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:16,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:17,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:21,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:21,340 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:21,341 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:21,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:21,347 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:21,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:23,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:25,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:25,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:25,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:25,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:25,818 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:26,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:27,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:30,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:30,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:30,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:30,362 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:30,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:30,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:30,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:31,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:31,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:34,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:34,902 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:34,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:34,908 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:35,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:35,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:37,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:39,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:04:43,865 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:43,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:43,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:43,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:43,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:44,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:45,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:04:45,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:04:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:04:45,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:48,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:50,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:50,676 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:50,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:50,681 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:50,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:51,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:04:54,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:04:56,207 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:04:56,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:04:56,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:04:56,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:04:56,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:04:57,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:00,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:00,632 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:00,633 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:00,633 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:00,638 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:00,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:00,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:00,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:00,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:01,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:05,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:05,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:05,860 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:05,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:05,865 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:06,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:07,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:09,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:05:10,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:10,323 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:10,323 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:10,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:10,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:11,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:11,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:14,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:14,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:14,899 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:14,905 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:15,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:15,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:15,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:15,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:15,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:16,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:19,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:19,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:19,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:19,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:19,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:21,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:22,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:24,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:24,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:24,821 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:24,827 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:24,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:25,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:27,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:29,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:29,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:29,750 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:29,756 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:29,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:30,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:30,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:30,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:31,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:32,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:34,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:34,941 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:34,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:34,947 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:35,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:35,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:37,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:39,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:05:40,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:40,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:40,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:40,287 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:40,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:41,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:43,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:44,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:44,956 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:44,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:44,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:05:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:05:45,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:05:45,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:45,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:48,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:49,033 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:49,033 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:49,033 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:49,039 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:49,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:49,913 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:54,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:05:54,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:54,625 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:54,625 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:54,630 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:54,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:05:55,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:59,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:05:59,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:05:59,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:05:59,060 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:05:59,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:05:59,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:00,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:00,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:00,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:00,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:04,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:04,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:04,575 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:04,580 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:04,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:05,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:05,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:09,806 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:09,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:09,807 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:09,812 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:09,867 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:06:09,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:10,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:11,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:14,361 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:14,362 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:14,362 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:14,367 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:14,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:15,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:15,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:15,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:15,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:20,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:23,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:23,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:23,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:23,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:23,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:23,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:26,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:27,374 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:27,375 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:27,375 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:27,380 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:27,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:27,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:30,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:31,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:31,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:31,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:31,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:31,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:32,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:33,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:37,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:38,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:38,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:38,681 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:38,687 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:38,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:39,869 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:06:39,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:42,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:06:45,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:06:45,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:06:47,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:47,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:47,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:47,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:47,915 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:47,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:49,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:52,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:52,468 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:52,468 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:52,474 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:52,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:53,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:53,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:06:58,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:06:58,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:06:58,631 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:06:58,631 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:06:58,636 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:06:58,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:06:59,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:00,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:00,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:03,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:04,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:04,359 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:04,359 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:04,364 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:04,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:05,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:09,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:09,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:09,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:09,834 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:09,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:09,871 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:07:09,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:12,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:14,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:14,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:14,117 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:14,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:15,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:15,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:15,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:15,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:16,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:19,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:19,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:19,782 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:19,788 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:20,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:20,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:22,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:24,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:24,870 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:24,871 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:24,876 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:25,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:25,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:26,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:29,254 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:29,255 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:29,255 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:29,260 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:30,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:30,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:30,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:30,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:30,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:35,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:37,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:37,803 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:37,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:37,808 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:38,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:39,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:07:40,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:41,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:41,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:41,921 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:41,922 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:41,927 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:42,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:44,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:07:45,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:07:45,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:07:46,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:46,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:46,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:46,744 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:47,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:47,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:48,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:50,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:50,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:50,868 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:50,873 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:51,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:07:52,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:07:52,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:57,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:07:58,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:07:58,945 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:07:58,945 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:07:58,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:07:59,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:00,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:00,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:00,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:02,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:04,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:04,412 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:04,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:04,417 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:05,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:06,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:08,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:09,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:09,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:09,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:09,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:09,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:08:10,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:10,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:13,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:13,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:13,609 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:13,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:13,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:14,068 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:15,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:15,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:16,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:18,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:19,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:19,740 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:19,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:19,745 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:20,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:22,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:24,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:25,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:25,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:25,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:25,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:26,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:26,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:29,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:29,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:29,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:29,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:30,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:30,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:30,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:30,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:32,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:34,842 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:34,843 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:34,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:34,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:35,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:36,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:36,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:39,847 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:39,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:39,848 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:39,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:39,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:08:40,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:42,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:42,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:45,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:45,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:45,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:45,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:45,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:08:45,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:08:45,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:08:46,104 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:46,104 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:47,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:50,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:50,074 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:50,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:50,079 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:50,106 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:52,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:53,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:08:54,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:08:54,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:08:54,533 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:08:54,538 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:08:55,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:08:56,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:08:58,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:00,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:00,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:00,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:00,941 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:00,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:00,946 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:01,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:02,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:03,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:08,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:09,331 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:09,332 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:09,333 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:09,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:09,882 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:09:10,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:10,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:14,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:14,044 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:14,044 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:14,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:14,049 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:14,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:15,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:15,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:15,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:16,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:18,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:18,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:18,491 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:18,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:19,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:19,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:20,138 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:22,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:22,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:22,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:22,916 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:23,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:24,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:24,917 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:28,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:28,161 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:28,162 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:28,167 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:29,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:30,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:30,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:30,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:30,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:35,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:36,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:36,664 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:36,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:36,670 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:37,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:38,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:39,882 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:09:41,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:41,378 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:41,379 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:41,379 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:41,384 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:42,160 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:42,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:45,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:09:45,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:09:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:09:46,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:47,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:47,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:47,339 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:47,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:48,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:48,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:51,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:51,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:51,866 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:51,871 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:52,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:09:52,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:54,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:09:57,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:09:58,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:09:58,427 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:09:58,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:09:58,432 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:09:59,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:00,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:00,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:00,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:02,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:04,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:04,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:04,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:04,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:04,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:06,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:08,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:09,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:10:10,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:10,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:10,116 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:10,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:10,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:12,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:13,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:14,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:14,113 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:14,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:14,119 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:14,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:15,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:15,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:15,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:16,193 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:18,515 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:18,516 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:18,516 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:18,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:18,521 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:19,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:20,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:22,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:22,895 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:22,895 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:22,900 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:23,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:23,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:24,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:27,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:27,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:27,361 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:27,366 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:28,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:28,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:29,367 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:30,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:30,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:32,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:32,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:32,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:32,690 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:33,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:34,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:34,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:37,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:37,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:37,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:37,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:38,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:38,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:39,886 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:10:39,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:41,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:41,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:41,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:41,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:42,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:42,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:45,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:45,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:10:45,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:10:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:10:47,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:47,242 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:47,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:47,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:48,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:48,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:51,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:51,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:51,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:51,512 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:51,517 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:52,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:52,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:10:56,518 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:10:57,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:10:57,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:10:57,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:10:57,687 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:10:58,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:10:58,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:00,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:00,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:00,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:02,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:02,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:02,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:02,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:02,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:03,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:04,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:07,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:07,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:07,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:07,293 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:07,298 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:08,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:08,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:09,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:11:12,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:15,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:15,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:16,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:16,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:16,202 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:16,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:16,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:18,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:18,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:20,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:20,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:20,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:20,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:21,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:22,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:23,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:28,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:29,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:29,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:29,290 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:29,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:29,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:30,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:30,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:30,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:33,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:34,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:34,146 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:34,146 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:34,151 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:34,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:36,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:38,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:38,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:38,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:38,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:39,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:39,894 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:11:39,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:40,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:44,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:44,457 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:44,457 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:44,462 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:45,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:45,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:45,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:11:45,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:11:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:11:46,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:49,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:49,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:49,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:49,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:49,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:50,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:51,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:53,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:53,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:53,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:53,904 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:54,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:11:56,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:11:56,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:11:58,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:11:58,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:11:58,323 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:11:58,327 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:11:58,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:00,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:00,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:01,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:03,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:03,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:03,218 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:03,223 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:03,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:04,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:07,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:08,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:08,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:08,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:08,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:09,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:09,892 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:12:10,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:12,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:13,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:13,112 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:13,113 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:13,118 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:13,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:14,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:15,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:15,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:17,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:22,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:22,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:22,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:22,577 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:23,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:23,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:24,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:27,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:27,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:27,210 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:27,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:27,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:28,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:29,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:30,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:30,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:30,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:34,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:34,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:34,511 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:34,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:34,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:35,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:36,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:39,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:39,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:39,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:39,883 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:39,887 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:39,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:12:40,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:40,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:44,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:45,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:12:45,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:12:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:12:48,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:48,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:48,279 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:48,284 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:48,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:50,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:50,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:53,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:53,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:53,600 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:53,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:12:54,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:12:54,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:12:55,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:12:59,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:12:59,703 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:12:59,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:12:59,709 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:00,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:00,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:00,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:00,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:00,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:04,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:04,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:04,610 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:04,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:05,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:06,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:06,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:09,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:09,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:09,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:09,105 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:09,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:09,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:13:10,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:11,899 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:15,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:15,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:15,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:15,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:15,700 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:15,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:16,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:16,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:16,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:21,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:21,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:21,023 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:21,028 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:21,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:22,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:22,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:26,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:26,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:26,218 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:26,223 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:26,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:27,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:28,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:30,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:30,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:30,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:31,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:31,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:31,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:31,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:32,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:32,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:32,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:35,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:35,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:35,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:35,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:36,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:36,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:37,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:39,899 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:13:42,357 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:42,358 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:42,358 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:42,363 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:42,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:43,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:44,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:45,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:13:45,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:13:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:13:48,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:48,548 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:48,548 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:48,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:48,553 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:49,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:50,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:52,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:52,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:52,863 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:52,868 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:53,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:53,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:13:54,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:57,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:13:57,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:13:57,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:13:57,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:13:57,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:13:58,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:13:59,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:00,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:00,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:00,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:01,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:01,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:01,643 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:01,648 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:02,483 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:02,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:04,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:07,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:07,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:07,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:07,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:07,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:08,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:09,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:14:09,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:13,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:13,711 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:13,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:13,716 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:14,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:14,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:15,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:15,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:15,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:15,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:20,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:22,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:22,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:22,117 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:22,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:22,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:24,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:26,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:26,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:26,620 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:26,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:26,625 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:27,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:28,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:30,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:30,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:31,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:31,751 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:31,751 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:31,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:31,757 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:32,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:32,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:36,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:36,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:36,474 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:36,480 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:36,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:37,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:38,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:39,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:14:40,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:40,860 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:40,860 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:40,865 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:41,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:42,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:42,866 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:44,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:44,863 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:44,863 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:44,868 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:45,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:14:45,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:14:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:14:46,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:47,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:49,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:49,406 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:49,406 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:49,411 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:49,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:50,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:53,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:54,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:54,063 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:54,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:54,069 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:54,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:14:56,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:14:59,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:14:59,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:14:59,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:14:59,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:14:59,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:14:59,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:00,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:00,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:00,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:00,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:05,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:07,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:07,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:07,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:07,661 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:08,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:08,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:09,909 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:15:10,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:12,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:12,393 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:12,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:12,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:12,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:14,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:15,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:15,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:15,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:15,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:16,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:16,500 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:16,500 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:16,505 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:16,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:18,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:21,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:23,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:23,162 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:23,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:23,167 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:23,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:24,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:27,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:30,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:32,147 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:32,148 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:32,148 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:32,153 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:32,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:32,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:34,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:37,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:38,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:38,216 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:38,216 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:38,221 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:38,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:39,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:15:40,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:42,442 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:42,443 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:42,443 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:42,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:42,448 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:42,596 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:44,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:15:45,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:15:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:15:47,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:47,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:47,291 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:47,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:47,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:48,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:48,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:52,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:52,026 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:52,026 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:52,031 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:52,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:52,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:54,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:15:56,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:15:56,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:15:56,373 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:15:56,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:15:56,610 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:15:58,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:15:59,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:00,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:00,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:00,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:01,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:01,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:01,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:01,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:01,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:02,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:04,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:09,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:09,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:09,629 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:09,629 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:09,634 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:09,913 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:16:10,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:10,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:14,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:14,453 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:14,454 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:14,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:14,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:14,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:15,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:15,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:15,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:16,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:19,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:19,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:19,155 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:19,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:19,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:20,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:20,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:23,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:23,577 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:23,577 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:23,582 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:23,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:24,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:25,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:30,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:30,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:30,505 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:30,510 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:30,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:30,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:30,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:30,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:30,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:32,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:35,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:35,710 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:35,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:35,715 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:36,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:36,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:36,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:39,912 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:16:40,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:40,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:40,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:40,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:40,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:42,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:42,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:45,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:16:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:16:45,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:16:45,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:45,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:45,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:45,948 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:46,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:46,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:47,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:50,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:50,880 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:50,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:50,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:51,665 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:52,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:53,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:16:56,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:16:56,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:16:56,186 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:16:56,190 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:16:56,670 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:16:58,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:16:59,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:00,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:00,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:00,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:02,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:02,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:02,262 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:02,267 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:02,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:04,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:04,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:06,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:06,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:06,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:06,750 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:07,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:08,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:09,750 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:09,914 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:17:11,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:11,349 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:11,349 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:11,354 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:11,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:12,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:15,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:15,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:15,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:15,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:16,426 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:16,427 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:16,427 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:16,432 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:16,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:18,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:20,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:20,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:20,963 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:20,964 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:20,969 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:21,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:22,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:25,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:27,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:27,207 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:27,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:27,213 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:27,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:28,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:30,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:30,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:30,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:31,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:32,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:32,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:32,092 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:32,097 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:32,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:32,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:37,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:37,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:37,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:37,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:37,059 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:37,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:38,718 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:39,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:17:41,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:41,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:41,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:41,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:41,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:42,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:42,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:17:45,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:17:45,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:17:47,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:47,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:47,851 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:47,851 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:47,856 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:48,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:48,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:52,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:52,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:52,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:52,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:52,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:52,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:53,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:17:57,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:17:57,055 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:17:57,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:17:57,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:17:57,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:17:58,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:17:59,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:00,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:00,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:00,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:04,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:04,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:04,601 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:04,601 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:04,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:04,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:06,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:09,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:09,918 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:18:09,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:09,942 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:09,942 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:09,947 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:10,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:10,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:14,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:15,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:15,305 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:15,305 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:15,310 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:15,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:15,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:15,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:15,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:16,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:19,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:20,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:20,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:20,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:20,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:20,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:22,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:25,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:25,870 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:25,871 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:25,871 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:25,876 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:26,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:26,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:30,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:30,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:30,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:30,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:30,932 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:30,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:30,932 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:30,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:31,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:32,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:35,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:35,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:35,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:35,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:35,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:36,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:36,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:39,921 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:18:41,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:41,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:41,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:41,805 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:41,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:42,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:42,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:18:45,702 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:18:45,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:18:46,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:46,062 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:46,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:46,067 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:46,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:46,790 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:47,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:51,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:51,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:51,103 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:51,108 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:51,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:52,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:52,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:18:56,359 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:18:56,360 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:18:56,360 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:18:56,365 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:18:56,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:18:57,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:18:58,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:00,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:00,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:00,339 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:00,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:00,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:00,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:00,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:00,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:02,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:02,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:04,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:04,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:04,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:04,974 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:05,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:06,836 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:08,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:09,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:09,903 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:09,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:09,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:09,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:19:10,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:10,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:13,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:14,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:14,204 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:14,205 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:14,209 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:14,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:14,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:15,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:15,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:15,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:18,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:18,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:18,542 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:18,547 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:18,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:19,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:20,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:24,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:25,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:25,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:25,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:25,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:26,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:26,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:29,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:30,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:30,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:30,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:30,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:30,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:30,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:30,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:30,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:32,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:34,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:36,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:36,248 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:36,248 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:36,253 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:36,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:36,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:39,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:19:40,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:40,634 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:40,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:40,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:40,639 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:40,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:42,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:45,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:45,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:19:45,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:19:45,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:19:46,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:46,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:46,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:46,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:46,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:48,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:50,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:55,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:55,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:55,284 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:55,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:55,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:19:56,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:19:56,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:19:59,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:19:59,283 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:19:59,283 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:19:59,288 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:19:59,887 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:00,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:00,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:00,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:00,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:01,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:05,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:05,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:05,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:05,855 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:05,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:06,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:07,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:09,927 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:20:11,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:11,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:11,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:11,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:12,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:12,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:12,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:15,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:15,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:15,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:18,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:19,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:19,870 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:19,870 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:19,876 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:19,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:20,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:23,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:23,713 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:23,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:23,718 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:23,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:24,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:24,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:27,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:27,959 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:27,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:27,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:28,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:28,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:29,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:30,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:30,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:30,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:32,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:32,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:32,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:32,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:32,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:32,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:35,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:36,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:36,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:36,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:36,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:36,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:38,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:39,932 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:20:40,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:41,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:41,932 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:41,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:41,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:41,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:42,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:45,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:20:45,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:20:45,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:20:45,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:50,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:50,080 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:50,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:50,085 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:50,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:50,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:51,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:55,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:55,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:55,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:55,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:55,951 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:20:56,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:20:56,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:20:59,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:20:59,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:20:59,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:20:59,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:20:59,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:00,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:00,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:00,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:00,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:01,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:04,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:04,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:04,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:04,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:04,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:06,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:06,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:09,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:09,420 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:09,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:09,425 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:09,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:21:09,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:10,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:12,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:15,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:15,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:15,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:16,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:16,161 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:16,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:16,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:16,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:16,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:18,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:20,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:20,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:20,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:20,428 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:20,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:20,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:23,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:25,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:25,038 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:25,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:25,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:25,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:26,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:29,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:30,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:30,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:30,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:30,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:31,002 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:31,002 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:31,007 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:31,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:32,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:35,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:38,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:38,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:38,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:38,673 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:39,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:39,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:21:40,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:41,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:42,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:42,900 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:42,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:42,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:43,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:45,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:45,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:21:45,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:21:45,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:21:45,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:47,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:47,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:47,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:47,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:48,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:49,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:21:51,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:54,847 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:21:54,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:21:54,848 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:21:54,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:21:55,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:21:56,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:21:57,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:00,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:00,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:00,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:00,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:00,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:00,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:00,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:01,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:01,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:03,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:06,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:06,004 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:06,004 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:06,009 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:06,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:07,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:07,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:09,935 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:22:10,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:10,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:10,491 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:10,496 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:11,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:11,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:12,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:15,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:15,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:15,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:17,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:18,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:18,890 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:18,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:18,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:19,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:21,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:23,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:23,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:23,993 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:23,994 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:23,998 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:24,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:25,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:28,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:28,405 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:28,405 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:28,411 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:29,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:29,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:29,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:30,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:30,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:30,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:33,259 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:33,260 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:33,260 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:33,265 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:34,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:35,058 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:35,304 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:38,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:38,007 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:38,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:38,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:38,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:39,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:39,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:22:40,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:42,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:42,264 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:42,264 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:42,269 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:43,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:43,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:45,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:22:45,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:22:45,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:22:45,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:46,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:46,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:46,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:46,652 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:47,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:49,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:51,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:52,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:52,076 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:52,076 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:52,081 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:52,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:53,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:22:56,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:22:56,811 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:22:56,812 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:22:56,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:22:56,817 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:22:57,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:22:59,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:00,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:00,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:01,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:01,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:01,547 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:01,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:02,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:02,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:03,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:06,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:06,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:06,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:06,910 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:07,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:07,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:09,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:09,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:23:10,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:10,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:10,975 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:10,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:11,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:12,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:13,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:15,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:15,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:15,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:17,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:17,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:17,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:17,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:18,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:18,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:19,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:23,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:25,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:25,524 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:25,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:25,530 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:26,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:27,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:28,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:30,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:30,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:30,098 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:30,102 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:30,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:30,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:30,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:30,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:31,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:34,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:35,243 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:35,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:35,244 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:35,249 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:36,148 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:37,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:39,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:39,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:39,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:39,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:39,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:39,942 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:23:40,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:41,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:44,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:45,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:23:45,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:23:45,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:23:46,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:46,452 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:46,452 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:46,457 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:47,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:47,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:50,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:51,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:51,602 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:51,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:51,609 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:52,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:53,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:23:55,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:23:57,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:23:57,521 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:23:57,521 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:23:57,527 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:23:58,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:23:59,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:00,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:00,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:01,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:02,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:02,777 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:02,777 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:02,782 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:03,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:05,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:06,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:09,082 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:09,083 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:09,083 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:09,088 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:09,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:09,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:24:11,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:11,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:15,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:15,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:15,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:15,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:15,997 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:15,998 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:16,003 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:16,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:17,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:17,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:20,465 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:20,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:20,466 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:20,471 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:21,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:21,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:22,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:24,886 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:24,887 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:24,887 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:24,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:25,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:27,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:27,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:30,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:30,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:30,221 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:30,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:30,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:30,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:30,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:31,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:31,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:32,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:34,865 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:34,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:34,866 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:34,871 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:35,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:37,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:38,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:39,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:39,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:39,145 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:39,150 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:39,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:39,946 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:24:41,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:43,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:43,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:43,943 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:43,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:43,948 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:44,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:45,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:45,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:24:45,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:24:45,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:24:49,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:50,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:50,237 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:50,237 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:50,242 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:51,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:51,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:54,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:24:54,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:24:54,672 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:24:54,672 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:24:54,677 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:24:55,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:24:55,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:24:59,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:00,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:00,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:00,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:00,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:00,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:00,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:00,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:01,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:04,565 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:04,566 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:04,566 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:04,571 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:05,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:05,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:05,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:08,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:08,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:08,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:08,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:09,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:09,951 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:25:10,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:11,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:15,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:15,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:15,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:16,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:16,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:16,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:16,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:16,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:17,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:19,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:21,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:22,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:22,740 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:22,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:22,745 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:23,264 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:23,264 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:27,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:30,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:30,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:30,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:31,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:31,004 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:31,004 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:31,009 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:31,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:33,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:33,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:36,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:36,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:36,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:36,298 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:37,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:37,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:38,298 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:39,950 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:25:40,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:40,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:40,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:40,938 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:41,280 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:43,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:43,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:45,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:45,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:45,293 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:45,305 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:45,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:25:45,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:25:45,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:25:46,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:47,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:48,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:51,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:51,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:51,042 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:51,047 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:51,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:53,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:25:54,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:57,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:25:57,089 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:25:57,089 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:25:57,094 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:25:57,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:25:59,095 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:25:59,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:00,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:00,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:01,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:01,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:01,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:01,758 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:02,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:03,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:04,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:06,361 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:06,362 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:06,362 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:06,367 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:07,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:07,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:09,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:26:09,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:10,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:10,592 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:10,593 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:10,598 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:11,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:11,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:15,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:15,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:15,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:15,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:19,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:19,094 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:19,094 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:19,100 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:19,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:21,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:21,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:24,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:24,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:24,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:24,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:24,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:25,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:26,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:28,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:28,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:28,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:28,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:29,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:29,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:30,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:30,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:30,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:32,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:33,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:33,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:33,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:33,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:34,336 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:35,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:37,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:39,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:39,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:39,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:39,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:39,958 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:26:40,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:41,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:42,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:44,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:44,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:44,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:44,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:45,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:45,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:45,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:26:45,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:26:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:26:47,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:49,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:49,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:49,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:49,176 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:49,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:51,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:53,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:53,491 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:53,492 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:53,492 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:53,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:54,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:26:55,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:26:58,498 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:26:59,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:26:59,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:26:59,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:26:59,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:26:59,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:00,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:00,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:00,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:01,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:03,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:03,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:03,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:03,308 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:03,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:04,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:05,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:07,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:07,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:07,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:07,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:08,372 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:09,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:09,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:09,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:27:14,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:15,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:15,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:15,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:15,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:15,982 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:15,983 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:15,988 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:16,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:17,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:19,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:20,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:20,425 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:20,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:20,431 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:21,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:21,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:25,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:25,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:25,519 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:25,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:25,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:26,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:27,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:30,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:30,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:30,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:30,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:32,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:32,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:32,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:32,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:33,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:33,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:36,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:37,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:37,580 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:37,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:37,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:38,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:39,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:39,961 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:27:41,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:45,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:27:45,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:27:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:27:46,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:46,076 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:46,076 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:46,081 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:46,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:47,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:47,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:50,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:50,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:50,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:50,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:51,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:51,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:52,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:27:56,297 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:27:56,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:27:56,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:27:56,305 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:27:56,436 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:27:57,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:27:58,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:00,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:00,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:00,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:01,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:01,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:01,157 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:01,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:01,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:03,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:04,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:05,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:05,576 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:05,576 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:05,581 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:06,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:07,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:09,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:09,961 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:28:11,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:11,588 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:11,589 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:11,594 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:12,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:13,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:14,594 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:15,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:15,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:15,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:16,055 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:16,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:16,060 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:16,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:17,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:20,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:20,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:20,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:20,331 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:20,336 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:20,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:21,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:25,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:25,144 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:25,144 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:25,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:25,149 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:25,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:27,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:30,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:30,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:30,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:30,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:30,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:30,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:30,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:30,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:30,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:31,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:35,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:36,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:36,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:36,284 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:36,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:36,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:37,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:39,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:28:41,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:41,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:41,013 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:41,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:41,018 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:41,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:43,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:45,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:28:45,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:28:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:28:46,607 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:46,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:46,608 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:46,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:46,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:47,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:47,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:51,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:51,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:51,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:51,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:51,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:28:52,297 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:53,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:28:57,298 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:28:59,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:28:59,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:28:59,379 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:28:59,384 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:28:59,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:00,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:00,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:00,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:01,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:03,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:03,853 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:03,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:03,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:03,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:04,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:05,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:08,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:08,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:08,346 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:08,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:08,350 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:08,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:09,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:09,965 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:29:13,607 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:13,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:13,608 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:13,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:13,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:14,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:15,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:15,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:15,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:15,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:18,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:18,626 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:18,626 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:18,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:18,631 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:19,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:19,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:23,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:26,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:26,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:26,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:26,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:27,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:27,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:28,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:30,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:30,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:30,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:32,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:32,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:32,015 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:32,020 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:32,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:33,607 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:34,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:36,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:36,261 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:36,262 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:36,266 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:36,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:37,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:39,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:39,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:29:40,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:40,942 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:40,942 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:40,947 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:41,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:41,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:44,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:45,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:45,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:45,182 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:45,187 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:45,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:45,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:29:45,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:29:45,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:29:47,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:50,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:51,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:51,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:51,444 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:51,449 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:51,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:53,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:29:55,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:29:57,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:29:57,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:29:57,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:29:57,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:29:58,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:29:59,649 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:00,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:00,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:00,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:00,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:02,595 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:02,596 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:02,596 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:02,601 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:02,651 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:03,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:06,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:07,293 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:07,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:07,294 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:07,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:07,663 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:09,668 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:09,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:30:11,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:13,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:13,819 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:13,819 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:13,824 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:14,674 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:15,678 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:15,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:15,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:15,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:17,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:18,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:18,624 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:18,624 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:18,630 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:18,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:19,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:22,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:24,293 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:24,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:24,294 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:24,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:24,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:25,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:28,299 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:29,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:29,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:29,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:29,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:30,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:30,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:30,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:30,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:31,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:34,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:36,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:36,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:36,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:36,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:37,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:37,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:39,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:39,972 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:30:41,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:41,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:41,023 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:41,028 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:41,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:41,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:45,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:45,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:30:45,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:30:45,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:30:46,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:46,082 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:46,082 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:46,087 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:46,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:47,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:50,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:50,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:50,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:50,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:50,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:50,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:51,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:55,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:55,116 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:55,116 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:55,121 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:55,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:30:55,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:30:56,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:30:59,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:30:59,588 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:30:59,588 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:30:59,593 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:30:59,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:00,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:00,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:00,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:01,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:01,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:05,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:05,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:05,098 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:05,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:05,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:05,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:07,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:09,978 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:31:10,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:10,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:10,601 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:10,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:10,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:11,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:12,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:15,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:15,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:15,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:15,045 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:15,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:15,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:15,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:15,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:15,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:17,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:20,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:20,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:20,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:20,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:21,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:22,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:23,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:24,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:24,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:24,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:24,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:25,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:26,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:28,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:29,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:29,020 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:29,021 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:29,026 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:30,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:30,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:30,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:30,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:30,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:33,558 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:33,559 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:33,559 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:33,564 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:34,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:34,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:36,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:39,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:39,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:39,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:39,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:39,690 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:39,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:31:40,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:42,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:44,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:45,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:31:45,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:31:45,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:31:48,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:48,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:48,006 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:48,011 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:48,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:50,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:50,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:52,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:52,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:52,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:52,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:53,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:54,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:31:55,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:31:56,491 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:31:56,492 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:31:56,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:31:56,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:31:57,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:31:58,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:00,498 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:00,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:00,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:00,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:01,506 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:01,507 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:01,507 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:01,512 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:02,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:04,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:05,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:06,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:06,359 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:06,359 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:06,365 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:07,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:08,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:09,981 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:32:10,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:11,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:11,535 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:11,536 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:11,540 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:12,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:14,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:15,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:15,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:15,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:16,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:17,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:17,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:17,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:17,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:18,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:20,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:22,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:22,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:22,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:22,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:22,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:23,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:24,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:27,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:29,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:29,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:29,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:29,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:30,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:30,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:30,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:30,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:32,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:32,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:33,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:33,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:33,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:33,993 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:34,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:36,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:38,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:39,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:39,047 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:39,047 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:39,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:39,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:39,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:32:40,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:43,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:44,209 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:44,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:44,210 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:44,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:45,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:45,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:32:45,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:32:45,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:32:46,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:48,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:48,874 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:48,874 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:48,879 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:49,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:49,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:50,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:53,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:53,945 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:53,945 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:53,950 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:54,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:32:54,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:32:56,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:32:58,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:32:58,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:32:58,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:32:58,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:32:59,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:00,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:00,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:00,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:00,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:00,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:02,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:02,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:02,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:02,700 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:03,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:04,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:05,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:09,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:09,168 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:09,168 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:09,173 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:09,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:33:10,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:10,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:10,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:14,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:14,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:14,082 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:14,087 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:14,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:15,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:15,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:15,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:16,131 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:16,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:19,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:19,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:19,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:19,136 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:19,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:20,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:22,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:23,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:23,572 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:23,573 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:23,578 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:24,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:26,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:27,578 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:29,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:29,861 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:29,861 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:29,867 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:30,148 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:30,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:30,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:30,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:32,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:32,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:35,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:35,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:35,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:35,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:36,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:38,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:38,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:39,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:33:41,149 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:41,150 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:41,150 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:41,155 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:41,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:42,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:44,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:45,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:33:45,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:33:45,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:33:49,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:49,438 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:49,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:49,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:49,443 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:50,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:52,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:54,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:54,053 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:54,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:54,059 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:54,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:33:55,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:33:56,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:33:58,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:33:58,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:33:58,474 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:33:58,479 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:33:59,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:00,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:00,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:00,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:00,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:05,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:05,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:05,575 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:05,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:05,580 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:06,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:08,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:09,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:34:10,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:11,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:11,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:11,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:11,445 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:12,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:14,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:15,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:15,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:15,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:15,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:17,525 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:17,526 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:17,526 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:17,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:18,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:20,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:21,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:22,632 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:22,633 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:22,633 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:22,638 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:23,199 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:24,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:26,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:27,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:27,068 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:27,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:27,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:27,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:28,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:30,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:30,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:30,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:31,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:31,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:31,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:31,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:32,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:32,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:34,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:36,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:36,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:36,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:36,986 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:37,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:37,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:38,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:39,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:34:42,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:42,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:42,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:42,920 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:43,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:43,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:44,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:45,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:34:45,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:34:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:34:47,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:47,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:47,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:47,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:48,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:49,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:50,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:51,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:51,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:51,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:51,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:52,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:54,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:34:54,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:34:56,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:34:56,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:34:56,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:34:56,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:34:57,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:34:58,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:00,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:00,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:00,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:01,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:01,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:01,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:01,279 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:02,243 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:02,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:06,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:06,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:06,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:06,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:06,436 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:07,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:08,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:09,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:35:11,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:11,027 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:11,027 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:11,032 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:11,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:12,033 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:12,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:15,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:15,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:15,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:16,306 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:16,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:16,307 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:16,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:17,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:17,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:18,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:20,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:20,415 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:20,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:20,421 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:21,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:22,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:22,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:24,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:24,633 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:24,634 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:24,638 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:25,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:26,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:27,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:29,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:29,091 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:29,092 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:29,097 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:29,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:30,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:30,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:30,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:30,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:32,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:34,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:34,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:34,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:34,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:35,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:36,280 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:38,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:39,315 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:39,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:39,316 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:39,321 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:39,996 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:35:40,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:40,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:43,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:44,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:44,143 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:44,144 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:44,149 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:44,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:45,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:35:45,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:35:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:35:46,291 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:48,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:48,781 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:48,781 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:48,786 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:49,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:49,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:35:50,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:52,673 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:35:52,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:35:52,674 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:35:52,679 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:35:53,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:35:54,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:35:55,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:00,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:00,017 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:00,017 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:00,022 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:00,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:00,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:00,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:00,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:02,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:04,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:04,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:04,225 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:04,230 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:04,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:06,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:06,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:08,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:08,644 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:08,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:08,649 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:09,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:09,995 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:36:10,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:11,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:13,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:13,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:13,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:13,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:13,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:14,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:15,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:15,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:15,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:17,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:17,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:17,453 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:17,454 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:17,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:18,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:18,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:22,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:22,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:22,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:22,928 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:22,933 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:23,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:24,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:27,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:27,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:27,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:27,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:28,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:28,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:28,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:30,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:30,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:30,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:32,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:32,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:32,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:32,376 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:33,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:34,339 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:34,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:37,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:37,438 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:37,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:37,443 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:38,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:38,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:39,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:40,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:36:41,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:41,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:41,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:41,952 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:42,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:44,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:44,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:36:45,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:36:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:36:46,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:46,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:46,253 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:46,258 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:36:46,350 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:36:48,354 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:36:50,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:55,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:36:59,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:36:59,607 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:36:59,607 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:36:59,612 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:00,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:00,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:00,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:00,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:00,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:02,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:04,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:04,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:04,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:04,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:05,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:05,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:06,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:09,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:09,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:09,298 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:09,303 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:09,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:09,999 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:37:10,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:11,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:13,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:13,982 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:13,982 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:13,988 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:14,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:15,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:15,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:15,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:16,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:16,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:18,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:18,362 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:18,363 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:18,368 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:18,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:20,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:21,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:23,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:23,002 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:23,002 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:23,007 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:23,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:24,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:27,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:27,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:27,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:27,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:27,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:27,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:28,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:30,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:30,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:30,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:32,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:32,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:32,053 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:32,053 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:32,058 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:32,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:34,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:37,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:37,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:37,339 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:37,339 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:37,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:37,452 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:38,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:40,005 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:37:42,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:42,280 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:42,281 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:42,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:42,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:42,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:44,461 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:37:45,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:37:45,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:37:47,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:47,573 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:47,574 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:47,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:47,579 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:48,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:37:48,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:52,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:55,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:37:55,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:37:55,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:37:55,970 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:37:56,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:37:57,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:37:58,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:00,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:00,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:00,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:01,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:01,087 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:01,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:01,093 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:01,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:02,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:03,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:06,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:06,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:06,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:06,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:07,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:08,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:08,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:10,004 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:38:11,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:11,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:11,463 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:11,468 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:11,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:12,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:14,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:15,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:15,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:15,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:18,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:18,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:18,975 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:18,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:19,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:20,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:20,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:23,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:23,673 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:23,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:23,678 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:24,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:24,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:25,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:30,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:30,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:30,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:30,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:30,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:30,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:30,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:30,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:31,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:32,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:35,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:35,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:35,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:35,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:35,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:36,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:36,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:40,005 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:38:42,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:43,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:43,782 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:43,782 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:43,787 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:44,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:45,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:38:45,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:38:45,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:38:46,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:47,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:49,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:49,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:49,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:49,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:50,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:50,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:53,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:38:54,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:38:54,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:38:54,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:38:54,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:38:54,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:38:56,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:38:59,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:00,154 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:00,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:00,155 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:00,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:00,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:00,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:00,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:00,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:02,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:04,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:05,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:05,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:05,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:05,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:06,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:08,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:10,011 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:39:10,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:10,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:10,941 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:10,942 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:10,947 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:11,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:12,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:15,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:15,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:15,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:15,051 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:15,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:15,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:15,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:15,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:15,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:16,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:20,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:20,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:20,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:20,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:20,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:20,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:22,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:25,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:26,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:26,277 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:26,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:26,282 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:26,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:28,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:30,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:30,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:30,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:30,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:32,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:32,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:32,984 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:32,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:33,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:34,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:36,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:40,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:39:40,449 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:40,450 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:40,450 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:40,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:40,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:41,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:42,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:45,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:39:45,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:39:45,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:39:46,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:46,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:46,933 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:46,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:46,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:47,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:48,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:51,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:51,405 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:51,405 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:51,410 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:51,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:52,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:52,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:39:55,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:39:55,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:39:55,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:39:55,859 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:39:56,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:39:57,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:39:58,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:00,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:00,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:00,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:00,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:00,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:00,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:00,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:00,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:02,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:03,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:06,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:06,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:06,638 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:06,643 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:06,650 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:08,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:08,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:10,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:40:11,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:11,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:11,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:11,545 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:11,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:12,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:14,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:15,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:15,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:15,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:18,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:18,529 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:18,529 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:18,535 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:18,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:19,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:20,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:23,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:23,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:23,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:23,986 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:24,668 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:24,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:26,671 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:29,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:29,710 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:29,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:29,715 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:30,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:30,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:30,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:30,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:30,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:30,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:34,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:34,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:34,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:34,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:35,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:36,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:36,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:40,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:40:41,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:41,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:41,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:41,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:41,685 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:42,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:42,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:45,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:40:45,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:40:45,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:40:46,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:46,935 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:46,935 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:46,941 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:47,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:47,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:48,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:51,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:51,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:51,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:51,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:51,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:52,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:53,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:40:56,314 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:40:56,315 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:40:56,315 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:40:56,320 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:40:56,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:40:58,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:40:59,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:00,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:00,455 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:00,456 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:00,461 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:00,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:00,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:00,710 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:00,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:02,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:05,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:07,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:07,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:07,265 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:07,270 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:07,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:08,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:10,019 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:41:10,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:11,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:11,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:11,368 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:11,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:11,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:12,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:15,373 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:15,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:15,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:15,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:15,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:15,926 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:15,926 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:15,931 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:16,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:18,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:20,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:20,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:20,291 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:20,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:20,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:21,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:22,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:26,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:26,002 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:26,003 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:26,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:26,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:27,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:28,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:30,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:30,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:30,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:32,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:34,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:34,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:34,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:34,711 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:34,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:36,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:38,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:38,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:38,954 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:38,955 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:38,959 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:39,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:40,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:41:40,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:44,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:44,782 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:44,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:44,783 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:44,788 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:45,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:41:45,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:41:45,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:41:45,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:46,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:49,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:51,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:51,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:51,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:51,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:51,754 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:52,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:41:55,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:41:56,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:41:56,538 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:41:56,538 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:41:56,543 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:41:56,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:41:58,763 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:00,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:00,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:00,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:00,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:00,995 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:00,995 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:01,000 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:01,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:02,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:06,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:09,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:09,862 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:09,862 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:09,867 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:10,021 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:42:10,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:10,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:11,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:13,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:13,748 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:13,749 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:13,753 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:13,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:14,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:15,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:15,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:15,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:16,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:18,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:18,746 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:18,746 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:18,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:18,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:20,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:22,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:23,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:23,211 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:23,211 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:23,216 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:23,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:24,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:28,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:28,120 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:28,120 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:28,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:28,125 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:28,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:30,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:30,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:30,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:30,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:32,750 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:32,751 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:32,751 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:32,756 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:32,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:33,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:34,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:36,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:36,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:36,989 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:36,995 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:37,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:38,806 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:38,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:40,023 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:42:43,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:43,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:43,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:43,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:43,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:44,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:44,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:45,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:42:45,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:42:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:42:48,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:48,935 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:48,935 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:48,940 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:49,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:49,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:50,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:54,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:54,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:54,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:54,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:54,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:42:55,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:42:56,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:42:59,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:42:59,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:42:59,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:42:59,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:42:59,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:00,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:00,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:00,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:00,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:00,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:05,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:06,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:06,200 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:06,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:06,206 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:06,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:08,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:10,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:43:11,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:13,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:13,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:13,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:13,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:13,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:14,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:15,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:15,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:15,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:17,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:17,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:17,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:17,777 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:17,782 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:17,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:18,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:22,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:23,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:23,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:23,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:23,105 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:23,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:24,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:28,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:28,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:28,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:28,647 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:28,652 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:28,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:30,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:30,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:30,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:30,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:33,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:35,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:35,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:35,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:35,739 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:35,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:36,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:39,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:39,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:39,961 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:39,961 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:39,966 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:40,031 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:43:40,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:40,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:44,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:44,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:44,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:44,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:44,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:45,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:45,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:43:45,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:43:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:43:46,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:48,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:48,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:48,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:48,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:48,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:50,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:50,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:52,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:52,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:52,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:52,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:52,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:54,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:43:55,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:43:56,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:43:56,748 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:43:56,748 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:43:56,753 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:43:56,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:43:58,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:00,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:00,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:00,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:01,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:06,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:06,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:06,646 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:06,646 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:06,652 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:06,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:08,894 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:10,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:44:11,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:13,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:13,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:13,284 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:13,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:13,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:14,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:15,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:15,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:15,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:16,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:17,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:17,624 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:17,624 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:17,629 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:17,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:18,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:21,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:21,845 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:21,845 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:21,850 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:21,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:22,851 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:22,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:26,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:26,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:26,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:26,113 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:26,913 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:28,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:28,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:30,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:30,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:30,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:30,428 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:30,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:30,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:30,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:30,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:32,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:34,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:35,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:35,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:35,328 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:35,333 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:35,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:36,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:39,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:39,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:39,984 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:39,984 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:39,989 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:40,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:44:40,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:40,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:45,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:45,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:45,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:45,132 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:45,137 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:44:45,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:44:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:44:45,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:46,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:50,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:51,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:51,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:51,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:51,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:51,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:52,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:44:56,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:44:56,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:44:56,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:44:56,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:44:56,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:44:56,943 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:44:58,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:00,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:00,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:00,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:01,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:01,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:01,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:01,067 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:01,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:02,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:02,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:05,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:05,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:05,658 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:05,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:05,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:06,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:07,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:10,034 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:45:10,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:10,983 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:10,983 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:10,989 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:11,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:12,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:12,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:15,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:15,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:15,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:16,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:16,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:16,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:16,750 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:16,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:18,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:18,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:21,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:21,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:21,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:21,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:21,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:22,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:24,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:26,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:26,962 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:26,962 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:26,968 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:26,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:28,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:29,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:30,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:30,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:30,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:31,461 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:31,462 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:31,462 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:31,482 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:31,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:32,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:35,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:35,761 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:35,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:35,763 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:35,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:35,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:36,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:40,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:45:40,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:40,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:40,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:40,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:40,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:41,297 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:42,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:44,699 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:44,700 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:44,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:44,705 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:44,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:45:45,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:45:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:45:46,996 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:47,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:49,239 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:49,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:49,240 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:49,245 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:49,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:51,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:52,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:54,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:54,270 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:54,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:54,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:45:55,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:45:57,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:45:58,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:45:59,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:45:59,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:45:59,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:45:59,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:00,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:00,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:00,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:00,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:01,015 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:03,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:07,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:07,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:07,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:07,196 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:08,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:09,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:09,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:10,038 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:46:11,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:11,437 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:11,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:11,443 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:12,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:13,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:14,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:15,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:15,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:15,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:17,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:17,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:17,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:17,711 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:18,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:19,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:19,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:22,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:22,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:22,269 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:22,274 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:23,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:25,038 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:25,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:27,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:27,606 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:27,606 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:27,611 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:28,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:29,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:30,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:30,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:30,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:30,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:31,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:31,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:31,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:31,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:32,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:33,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:36,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:36,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:36,483 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:36,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:36,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:37,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:39,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:40,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:46:41,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:41,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:41,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:41,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:41,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:42,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:43,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:45,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:46:45,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:46:45,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:46:46,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:46,411 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:46,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:46,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:46,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:47,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:49,061 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:50,950 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:50,951 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:50,951 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:50,956 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:51,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:51,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:46:53,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:55,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:46:55,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:46:55,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:46:55,855 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:46:56,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:46:57,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:46:57,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:00,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:00,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:00,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:02,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:04,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:04,064 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:04,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:04,069 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:04,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:05,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:08,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:10,042 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:47:11,207 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:11,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:11,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:11,213 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:12,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:13,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:13,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:15,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:15,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:15,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:16,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:16,038 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:16,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:16,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:16,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:17,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:19,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:19,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:19,974 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:19,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:19,979 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:20,125 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:21,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:24,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:26,761 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:26,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:26,762 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:26,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:27,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:29,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:30,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:30,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:30,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:31,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:34,801 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:34,802 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:34,802 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:34,807 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:35,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:36,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:37,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:40,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:47:40,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:40,384 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:40,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:40,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:41,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:42,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:43,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:45,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:47:45,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:47:45,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:47:45,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:46,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:46,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:46,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:46,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:47,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:48,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:52,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:52,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:52,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:52,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:52,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:53,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:47:53,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:55,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:47:55,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:47:55,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:47:55,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:47:56,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:47:57,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:47:58,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:00,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:00,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:00,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:00,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:00,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:00,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:00,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:01,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:03,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:04,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:05,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:05,891 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:05,892 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:05,897 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:06,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:07,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:09,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:10,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:48:10,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:10,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:10,512 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:10,517 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:11,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:13,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:14,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:14,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:14,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:14,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:14,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:15,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:15,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:15,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:15,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:17,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:20,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:20,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:20,881 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:20,881 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:20,887 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:21,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:23,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:25,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:29,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:29,349 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:29,349 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:29,354 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:30,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:30,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:30,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:30,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:30,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:31,199 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:34,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:34,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:34,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:34,923 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:35,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:36,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:37,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:40,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:48:42,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:43,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:43,384 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:43,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:43,389 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:44,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:45,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:45,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:48:45,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:48:45,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:48:47,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:47,664 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:47,664 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:47,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:47,669 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:48,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:49,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:52,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:53,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:53,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:53,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:53,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:54,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:48:55,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:48:57,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:48:58,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:48:58,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:48:58,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:48:58,691 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:48:59,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:00,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:00,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:00,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:01,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:02,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:03,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:03,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:03,610 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:03,615 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:04,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:05,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:08,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:09,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:09,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:09,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:09,405 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:10,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:49:10,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:11,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:13,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:13,433 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:13,433 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:13,438 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:14,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:14,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:15,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:15,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:15,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:15,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:19,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:19,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:19,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:19,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:19,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:20,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:21,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:24,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:26,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:26,621 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:26,622 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:26,627 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:27,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:29,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:30,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:30,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:30,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:30,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:31,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:31,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:31,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:31,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:31,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:33,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:36,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:39,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:39,496 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:39,496 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:39,501 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:40,053 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:49:40,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:41,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:42,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:44,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:44,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:44,811 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:44,816 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:45,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:45,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:49:45,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:49:45,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:49:47,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:47,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:48,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:48,944 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:48,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:48,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:49,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:51,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:52,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:54,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:54,881 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:54,882 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:54,887 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:55,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:49:57,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:49:58,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:49:59,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:49:59,384 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:49:59,384 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:49:59,389 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:49:59,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:00,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:00,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:00,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:01,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:03,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:07,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:07,901 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:07,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:07,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:08,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:08,944 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:09,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:10,055 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:50:14,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:14,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:14,738 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:14,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:14,743 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:15,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:15,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:15,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:15,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:17,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:18,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:18,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:18,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:18,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:19,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:19,823 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:21,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:24,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:25,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:25,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:25,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:25,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:25,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:27,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:30,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:30,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:30,085 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:30,085 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:30,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:30,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:30,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:30,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:30,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:31,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:35,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:35,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:35,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:35,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:35,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:36,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:37,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:40,057 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:50:41,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:44,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:44,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:44,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:44,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:44,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:45,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:45,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:50:45,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:50:45,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:50:46,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:49,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:49,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:49,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:49,045 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:49,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:51,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:52,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:53,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:53,302 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:53,302 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:53,307 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:53,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:55,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:50:57,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:50:58,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:50:58,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:50:58,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:50:58,177 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:50:58,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:50:59,490 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:00,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:00,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:00,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:02,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:02,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:02,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:02,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:02,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:03,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:05,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:07,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:07,583 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:07,583 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:07,588 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:08,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:08,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:09,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:10,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:51:11,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:11,858 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:11,858 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:11,864 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:12,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:13,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:13,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:15,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:15,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:15,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:16,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:16,536 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:16,536 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:16,541 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:17,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:17,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:19,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:21,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:21,357 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:21,357 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:21,362 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:21,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:23,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:25,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:26,239 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:26,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:26,240 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:26,245 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:26,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:27,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:30,595 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:30,596 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:30,596 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:30,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:30,601 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:30,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:30,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:30,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:31,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:31,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:35,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:37,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:37,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:37,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:37,374 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:37,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:39,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:40,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:51:41,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:42,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:42,904 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:42,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:42,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:43,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:45,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:45,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:51:45,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:51:45,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:51:46,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:48,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:48,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:48,366 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:48,370 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:48,551 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:49,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:52,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:53,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:53,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:53,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:53,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:53,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:55,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:51:57,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:51:57,862 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:51:57,862 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:51:57,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:51:57,867 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:51:58,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:51:59,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:00,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:00,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:00,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:02,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:02,421 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:02,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:02,426 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:02,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:03,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:03,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:06,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:06,668 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:06,669 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:06,674 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:07,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:08,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:09,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:10,063 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:52:11,918 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:11,918 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:11,919 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:11,924 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:12,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:13,578 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:13,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:15,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:15,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:15,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:16,631 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:16,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:16,632 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:16,637 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:17,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:19,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:19,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:22,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:22,060 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:22,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:22,065 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:22,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:23,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:25,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:28,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:28,541 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:28,542 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:28,547 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:28,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:29,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:30,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:30,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:30,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:30,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:33,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:33,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:33,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:33,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:33,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:35,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:36,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:37,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:37,606 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:37,606 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:37,611 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:38,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:39,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:40,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:52:42,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:43,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:43,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:43,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:43,113 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:43,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:45,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:52:45,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:52:45,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:52:48,015 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:48,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:48,840 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:48,841 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:48,846 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:49,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:51,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:53,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:54,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:54,737 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:54,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:54,742 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:55,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:52:57,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:52:59,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:52:59,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:52:59,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:52:59,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:52:59,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:52:59,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:00,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:00,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:00,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:01,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:04,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:04,017 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:04,017 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:04,022 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:04,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:05,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:05,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:08,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:08,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:08,491 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:08,496 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:08,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:10,068 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:53:10,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:11,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:14,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:14,858 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:14,858 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:14,863 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:15,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:15,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:15,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:15,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:15,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:17,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:19,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:19,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:19,115 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:19,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:19,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:21,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:21,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:24,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:24,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:24,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:24,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:24,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:26,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:27,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:29,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:29,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:29,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:29,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:30,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:30,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:30,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:30,734 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:31,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:31,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:34,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:34,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:34,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:34,142 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:34,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:35,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:37,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:38,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:38,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:38,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:38,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:39,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:40,073 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:53:41,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:43,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:44,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:44,207 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:44,208 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:44,212 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:44,749 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:53:45,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:53:45,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:53:45,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:48,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:49,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:49,302 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:49,302 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:49,308 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:49,753 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:51,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:54,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:53:55,461 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:53:55,462 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:53:55,462 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:53:55,467 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:53:55,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:53:57,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:53:59,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:00,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:00,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:00,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:01,154 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:01,154 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:01,155 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:01,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:01,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:03,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:05,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:06,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:06,073 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:06,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:06,078 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:06,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:07,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:10,071 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:54:11,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:14,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:14,538 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:14,538 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:14,543 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:14,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:15,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:15,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:15,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:16,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:17,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:21,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:21,288 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:21,288 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:21,293 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:21,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:22,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:23,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:25,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:25,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:25,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:25,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:25,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:27,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:27,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:30,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:30,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:30,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:32,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:33,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:33,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:33,909 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:33,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:34,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:35,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:37,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:38,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:38,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:38,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:38,561 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:38,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:40,073 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:54:41,806 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:42,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:42,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:42,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:42,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:42,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:43,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:45,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:54:45,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:54:45,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:54:45,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:47,802 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:47,803 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:47,803 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:47,808 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:47,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:48,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:49,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:52,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:52,921 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:52,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:52,926 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:53,818 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:53,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:54:55,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:54:58,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:54:58,344 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:54:58,345 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:54:58,350 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:54:58,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:54:59,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:00,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:00,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:00,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:01,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:04,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:04,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:04,030 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:04,035 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:04,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:05,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:05,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:08,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:08,757 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:08,758 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:08,763 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:08,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:10,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:55:10,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:11,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:15,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:15,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:15,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:15,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:17,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:17,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:17,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:17,136 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:17,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:19,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:20,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:21,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:21,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:21,347 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:21,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:21,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:23,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:25,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:25,585 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:25,585 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:25,586 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:25,591 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:25,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:27,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:30,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:30,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:30,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:30,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:33,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:33,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:33,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:33,986 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:34,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:35,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:36,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:40,077 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:55:40,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:40,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:40,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:40,279 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:40,865 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:41,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:41,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:44,730 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:44,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:44,731 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:44,736 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:44,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:45,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:55:45,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:55:45,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:55:46,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:47,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:49,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:49,434 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:49,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:49,439 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:49,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:51,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:52,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:54,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:54,434 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:54,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:54,440 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:54,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:55:57,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:55:58,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:55:59,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:55:59,216 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:55:59,216 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:55:59,221 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:55:59,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:00,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:00,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:00,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:01,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:04,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:05,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:05,812 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:05,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:05,817 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:05,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:07,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:09,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:10,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:56:11,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:11,473 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:11,473 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:11,479 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:11,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:13,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:15,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:15,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:15,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:15,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:15,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:15,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:15,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:15,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:15,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:17,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:20,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:20,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:20,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:20,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:20,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:21,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:21,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:24,635 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:24,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:24,636 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:24,641 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:24,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:26,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:27,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:28,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:28,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:28,719 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:28,724 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:28,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:30,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:30,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:30,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:31,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:32,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:33,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:33,537 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:33,537 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:33,543 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:33,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:34,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:37,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:38,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:38,015 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:38,015 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:38,020 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:38,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:38,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:40,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:56:42,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:42,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:42,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:42,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:42,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:42,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:44,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:45,901 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:56:45,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:56:45,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:56:47,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:47,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:47,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:47,085 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:47,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:48,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:48,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:53,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:56:54,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:56:54,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:56:54,180 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:56:54,185 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:56:54,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:56:54,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:56:58,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:00,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:00,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:00,093 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:00,097 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:00,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:00,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:00,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:04,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:05,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:05,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:05,241 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:05,246 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:05,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:06,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:09,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:09,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:09,783 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:09,784 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:09,789 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:09,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:10,083 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:57:10,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:15,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:15,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:15,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:15,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:15,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:16,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:16,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:16,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:16,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:16,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:20,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:20,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:20,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:20,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:20,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:20,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:22,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:25,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:25,182 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:25,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:25,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:25,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:25,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:26,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:29,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:29,734 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:29,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:29,739 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:29,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:30,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:30,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:30,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:30,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:30,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:35,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:35,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:35,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:35,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:35,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:36,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:36,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:39,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:39,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:39,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:39,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:39,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:40,085 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:57:41,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:42,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:43,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:43,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:43,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:43,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:44,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:45,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:45,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:57:45,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:57:45,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:57:47,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:48,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:48,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:48,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:48,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:49,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:49,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:52,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:57:52,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:57:52,102 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:57:52,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:57:53,018 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:57:53,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:57:53,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:57:58,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:00,515 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:00,516 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:00,516 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:00,521 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:00,902 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:01,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:01,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:03,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:05,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:05,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:05,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:05,378 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:06,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:07,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:08,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:10,087 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:58:10,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:10,525 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:10,525 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:10,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:11,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:11,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:13,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:14,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:14,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:14,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:14,758 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:15,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:15,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:15,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:15,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:17,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:19,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:19,047 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:19,047 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:19,047 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:19,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:19,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:21,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:23,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:23,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:23,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:23,774 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:24,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:24,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:25,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:29,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:30,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:30,031 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:30,032 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:30,037 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:30,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:30,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:30,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:30,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:31,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:34,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:34,412 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:34,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:34,417 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:35,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:35,072 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:35,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:39,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:39,452 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:39,452 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:39,457 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:40,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:40,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:58:41,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:41,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:43,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:43,981 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:43,982 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:43,987 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:44,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:45,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:45,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:58:45,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:58:45,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:58:46,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:48,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:48,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:48,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:48,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:49,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:49,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:51,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:54,275 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:54,276 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:54,276 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:54,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:58:55,091 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:58:55,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:58:56,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:58:59,710 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:58:59,710 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:58:59,711 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:58:59,716 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:00,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:00,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:00,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:01,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:02,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:04,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:04,999 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:05,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:05,004 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:05,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:07,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:08,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:10,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:59:11,578 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:11,578 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:11,579 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:11,584 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:12,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:13,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:13,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:15,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:15,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:15,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:16,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:16,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:16,164 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:16,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:17,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:17,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:19,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:21,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:21,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:21,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:21,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:21,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:23,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:25,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:26,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:26,545 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:26,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:26,550 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:27,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:27,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:30,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:30,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:30,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:30,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:33,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:33,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:33,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:33,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:33,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:35,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:36,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:37,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:37,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:37,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:37,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:38,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:39,185 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:40,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 16:59:41,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:42,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:42,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:42,202 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:42,207 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:43,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:43,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:45,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 16:59:45,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 16:59:45,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 16:59:46,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:46,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:46,682 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:46,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:46,688 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:47,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:49,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:51,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:51,033 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:51,033 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:51,038 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:51,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:52,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 16:59:53,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:55,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 16:59:55,291 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 16:59:55,291 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 16:59:55,296 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 16:59:56,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 16:59:57,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 16:59:57,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:00,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:00,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:00,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:01,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:01,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:01,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:01,193 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:01,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:03,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:03,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:06,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:06,182 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:06,182 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:06,187 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:06,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:07,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:09,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:10,096 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:00:10,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:10,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:10,474 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:10,479 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:11,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:11,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:14,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:15,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:15,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:15,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:18,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:18,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:18,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:18,920 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:19,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:19,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:21,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:22,762 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:22,763 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:22,763 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:22,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:23,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:25,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:25,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:28,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:28,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:28,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:28,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:28,241 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:29,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:30,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:30,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:30,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:30,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:32,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:32,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:32,370 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:32,376 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:33,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:33,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:36,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:36,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:36,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:36,916 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:36,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:37,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:39,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:40,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:00:42,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:42,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:42,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:42,275 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:42,280 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:43,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:43,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:45,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:00:45,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:00:45,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:00:47,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:47,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:47,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:47,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:47,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:47,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:49,264 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:51,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:00:51,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:00:51,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:00:51,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:00:52,266 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:00:52,509 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:00:53,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:00:57,509 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:00,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:00,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:00,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:00,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:00,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:00,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:00,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:00,903 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:01,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:03,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:06,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:06,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:06,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:06,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:07,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:07,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:08,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:10,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:01:10,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:10,956 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:10,956 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:10,961 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:11,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:13,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:13,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:15,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:15,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:15,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:15,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:15,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:15,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:16,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:17,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:19,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:20,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:20,927 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:20,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:20,932 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:21,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:23,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:24,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:25,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:25,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:25,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:25,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:26,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:27,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:29,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:29,501 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:29,501 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:29,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:30,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:30,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:30,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:30,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:30,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:31,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:34,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:34,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:34,928 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:34,933 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:35,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:35,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:37,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:39,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:39,703 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:39,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:39,709 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:40,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:01:40,319 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:41,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:41,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:45,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:01:45,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:01:45,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:01:46,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:46,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:46,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:46,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:46,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:47,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:47,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:51,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:51,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:51,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:51,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:51,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:51,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:53,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:01:56,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:01:56,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:01:56,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:01:56,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:01:56,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:01:57,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:01:57,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:00,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:00,657 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:00,657 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:00,663 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:00,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:00,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:01,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:01,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:02,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:05,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:05,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:05,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:05,952 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:06,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:07,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:07,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:10,104 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:02:10,498 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:10,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:10,499 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:10,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:11,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:11,354 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:13,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:15,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:15,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:15,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:15,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:15,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:15,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:15,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:17,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:19,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:23,650 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:23,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:23,651 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:23,656 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:24,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:24,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:25,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:27,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:27,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:27,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:27,707 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:28,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:29,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:29,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:30,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:30,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:30,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:32,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:32,473 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:32,473 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:32,478 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:33,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:33,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:35,479 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:39,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:39,857 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:39,857 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:39,862 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:40,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:02:40,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:41,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:41,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:43,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:43,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:43,978 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:43,983 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:44,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:45,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:45,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:02:45,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:02:45,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:02:46,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:48,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:48,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:48,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:48,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:49,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:02:49,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:51,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:56,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:02:56,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:02:56,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:02:56,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:02:56,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:02:57,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:02:57,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:00,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:01,559 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:01,560 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:01,560 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:01,565 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:02,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:02,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:03,438 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:06,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:06,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:06,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:06,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:07,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:07,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:07,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:10,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:03:11,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:11,538 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:11,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:11,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:12,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:13,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:13,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:15,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:16,775 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:16,776 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:16,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:16,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:17,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:17,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:18,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:20,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:20,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:20,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:20,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:21,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:21,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:24,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:24,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:24,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:24,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:24,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:25,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:25,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:29,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:30,239 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:30,239 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:30,240 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:30,245 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:30,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:30,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:30,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:30,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:31,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:34,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:34,445 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:34,445 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:34,450 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:34,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:35,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:35,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:39,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:39,639 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:39,639 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:39,645 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:40,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:03:40,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:41,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:41,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:45,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:03:45,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:03:45,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:03:46,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:47,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:47,068 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:47,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:47,074 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:47,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:49,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:52,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:54,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:54,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:54,003 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:54,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:54,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:03:55,494 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:58,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:03:58,494 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:03:58,495 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:03:58,495 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:03:58,500 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:03:59,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:03:59,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:00,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:02,754 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:02,755 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:02,755 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:02,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:03,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:03,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:03,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:08,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:08,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:08,846 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:08,847 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:08,872 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:09,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:09,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:10,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:04:13,329 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:13,330 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:13,330 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:13,336 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:13,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:14,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:15,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:15,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:20,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:21,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:21,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:21,637 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:21,642 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:22,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:23,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:25,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:27,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:27,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:27,266 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:27,270 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:27,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:29,535 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:30,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:30,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:30,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:30,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:31,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:31,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:31,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:31,972 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:32,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:33,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:36,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:39,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:39,355 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:39,355 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:39,360 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:39,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:40,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:04:41,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:41,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:45,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:04:45,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:04:45,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:04:45,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:46,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:46,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:46,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:46,137 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:46,551 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:47,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:50,851 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:50,852 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:50,852 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:50,857 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:51,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:51,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:04:51,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:56,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:04:56,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:04:56,939 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:04:56,939 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:04:56,945 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:04:57,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:04:57,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:00,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:02,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:02,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:02,666 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:02,666 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:02,671 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:03,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:03,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:07,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:07,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:07,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:07,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:07,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:07,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:09,650 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:10,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:05:13,010 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:13,011 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:13,011 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:13,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:13,016 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:13,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:13,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:15,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:15,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:15,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:18,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:19,037 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:19,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:19,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:19,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:19,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:19,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:23,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:23,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:23,330 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:23,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:23,335 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:23,660 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:25,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:28,335 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:28,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:28,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:28,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:28,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:29,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:29,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:30,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:30,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:30,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:34,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:37,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:37,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:37,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:37,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:37,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:39,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:39,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:40,119 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:05:42,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:42,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:42,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:42,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:42,714 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:43,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:45,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:45,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:05:45,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:05:45,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:05:48,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:48,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:48,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:48,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:48,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:49,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:50,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:05:53,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:05:53,936 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:05:53,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:05:53,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:05:54,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:05:55,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:05:55,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:00,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:00,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:01,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:01,740 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:01,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:01,745 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:02,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:03,734 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:06,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:06,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:06,830 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:06,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:06,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:07,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:07,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:10,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:06:11,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:11,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:11,776 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:11,781 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:12,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:12,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:13,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:15,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:15,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:15,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:18,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:18,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:18,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:18,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:18,059 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:18,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:19,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:22,862 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:22,862 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:22,863 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:22,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:23,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:23,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:23,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:27,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:27,703 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:27,703 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:27,708 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:27,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:29,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:29,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:30,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:30,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:30,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:31,610 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:31,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:31,610 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:31,616 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:31,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:33,768 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:35,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:38,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:38,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:38,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:38,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:39,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:39,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:40,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:06:41,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:44,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:44,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:44,193 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:44,198 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:44,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:45,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:45,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:06:45,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:06:45,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:06:46,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:48,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:48,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:48,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:48,672 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:48,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:49,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:51,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:52,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:52,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:52,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:52,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:53,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:53,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:06:56,801 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:06:57,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:06:57,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:06:57,170 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:06:57,175 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:06:57,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:06:57,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:00,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:00,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:01,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:01,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:01,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:01,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:01,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:02,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:03,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:07,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:10,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:07:10,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:10,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:10,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:10,451 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:10,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:11,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:12,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:15,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:15,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:15,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:18,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:18,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:18,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:18,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:18,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:18,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:19,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:23,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:26,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:26,959 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:26,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:26,964 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:27,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:27,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:28,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:30,905 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:30,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:30,906 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:31,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:31,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:31,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:31,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:31,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:33,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:34,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:38,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:38,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:38,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:38,195 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:38,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:39,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:40,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:07:40,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:43,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:43,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:43,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:43,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:44,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:45,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:45,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:46,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:07:46,053 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:07:46,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:07:50,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:50,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:50,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:50,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:50,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:51,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:07:51,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:54,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:07:54,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:07:54,746 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:07:54,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:07:54,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:07:55,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:07:56,751 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:00,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:00,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:00,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:00,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:00,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:01,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:01,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:01,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:01,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:02,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:05,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:05,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:05,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:05,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:05,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:07,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:08,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:10,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:08:12,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:12,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:12,373 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:12,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:12,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:13,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:13,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:16,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:16,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:16,053 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:17,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:17,207 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:17,207 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:17,212 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:17,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:17,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:19,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:22,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:22,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:22,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:22,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:22,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:23,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:24,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:28,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:28,187 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:28,187 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:28,192 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:28,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:29,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:30,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:31,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:33,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:33,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:33,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:33,798 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:33,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:35,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:35,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:40,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:40,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:40,125 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:40,131 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:40,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:08:40,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:41,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:41,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:44,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:44,403 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:44,403 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:44,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:44,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:45,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:46,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:08:46,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:08:46,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:08:46,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:49,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:49,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:49,987 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:49,992 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:50,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:51,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:08:52,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:55,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:08:55,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:08:55,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:08:55,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:08:55,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:08:57,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:08:57,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:01,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:01,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:01,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:01,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:01,295 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:01,295 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:01,300 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:01,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:01,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:03,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:05,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:05,857 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:05,857 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:05,862 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:05,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:07,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:08,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:10,047 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:10,048 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:10,048 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:10,053 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:10,134 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:09:10,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:11,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:14,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:16,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:16,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:16,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:16,326 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:16,327 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:16,327 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:16,332 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:16,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:17,983 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:19,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:21,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:21,094 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:21,094 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:21,099 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:21,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:21,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:25,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:25,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:25,461 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:25,461 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:25,466 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:25,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:27,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:30,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:31,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:31,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:31,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:31,316 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:31,321 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:31,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:31,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:36,322 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:39,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:39,553 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:39,554 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:39,559 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:40,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:40,136 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:09:42,009 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:42,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:43,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:43,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:43,643 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:43,648 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:44,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:46,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:46,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:09:46,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:09:46,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:09:47,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:48,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:48,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:48,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:48,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:49,015 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:50,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:53,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:09:53,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:09:53,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:09:53,145 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:09:53,150 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:09:54,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:09:54,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:09:58,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:01,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:01,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:01,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:01,601 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:01,602 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:01,602 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:01,606 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:02,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:03,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:04,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:08,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:08,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:08,085 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:08,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:09,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:09,090 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:10,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:10,141 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:10:13,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:13,151 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:13,152 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:13,157 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:14,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:14,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:14,157 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:16,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:16,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:16,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:18,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:18,541 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:18,541 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:18,546 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:19,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:19,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:20,114 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:24,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:27,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:27,069 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:27,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:27,074 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:27,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:28,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:30,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:31,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:32,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:32,310 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:32,311 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:32,316 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:33,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:34,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:35,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:38,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:38,212 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:38,212 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:38,218 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:39,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:40,140 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:10:40,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:41,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:43,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:43,072 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:43,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:43,078 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:43,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:44,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:46,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:10:46,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:10:46,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:10:46,299 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:47,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:47,186 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:47,186 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:47,191 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:48,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:48,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:51,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:51,645 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:51,645 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:51,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:51,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:52,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:54,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:10:56,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:10:56,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:10:56,328 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:10:56,333 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:10:57,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:10:57,333 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:10:58,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:01,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:01,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:01,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:02,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:02,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:02,092 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:02,098 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:02,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:03,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:04,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:06,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:06,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:06,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:06,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:07,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:08,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:08,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:10,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:11:11,095 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:11,096 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:11,096 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:11,101 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:11,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:12,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:14,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:15,729 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:15,730 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:15,730 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:15,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:16,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:16,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:16,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:18,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:19,343 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:20,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:20,680 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:20,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:20,685 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:21,206 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:22,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:24,724 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:28,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:28,147 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:28,147 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:28,152 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:28,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:30,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:30,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:31,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:31,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:31,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:34,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:34,821 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:34,821 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:34,826 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:35,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:35,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:36,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:39,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:39,012 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:39,012 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:39,017 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:39,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:40,144 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:11:40,227 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:41,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:46,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:11:46,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:11:46,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:11:46,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:47,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:47,387 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:47,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:47,392 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:48,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:48,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:51,428 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:54,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:54,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:54,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:54,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:54,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:11:56,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:11:57,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:11:58,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:11:58,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:11:58,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:11:58,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:11:59,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:00,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:01,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:01,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:01,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:02,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:04,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:04,501 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:04,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:04,506 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:05,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:06,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:07,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:09,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:09,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:09,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:09,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:09,253 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:10,149 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:12:10,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:13,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:14,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:14,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:14,125 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:14,130 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:14,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:16,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:16,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:18,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:18,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:18,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:18,070 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:18,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:19,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:20,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:23,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:23,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:23,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:23,914 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:24,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:24,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:26,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:28,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:28,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:28,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:28,819 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:29,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:30,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:30,820 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:31,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:31,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:31,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:33,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:33,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:33,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:33,798 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:34,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:35,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:36,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:38,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:38,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:38,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:38,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:38,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:40,148 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:12:40,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:41,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:42,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:42,434 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:42,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:42,440 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:43,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:44,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:46,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:12:46,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:12:46,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:12:46,361 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:46,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:46,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:46,744 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:46,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:47,291 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:48,293 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:51,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:12:53,374 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:12:53,375 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:12:53,375 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:12:53,380 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:12:54,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:12:54,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:12:57,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:00,333 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:00,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:00,334 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:00,339 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:01,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:01,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:01,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:01,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:02,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:03,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:05,894 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:05,894 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:05,895 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:05,900 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:06,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:08,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:08,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:10,150 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:13:11,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:11,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:11,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:11,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:12,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:12,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:14,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:16,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:17,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:17,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:17,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:17,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:17,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:18,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:20,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:21,581 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:21,585 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:21,585 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:21,591 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:22,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:22,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:25,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:27,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:27,117 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:27,117 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:27,122 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:27,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:28,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:31,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:31,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:31,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:31,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:33,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:33,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:33,405 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:33,410 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:34,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:34,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:36,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:37,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:37,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:37,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:37,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:38,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:38,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:40,152 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:13:41,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:41,570 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:41,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:41,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:41,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:42,396 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:42,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:46,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:13:46,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:13:46,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:13:47,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:48,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:48,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:48,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:48,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:49,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:50,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:52,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:53,233 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:53,234 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:53,234 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:53,239 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:53,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:13:54,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:13:58,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:13:58,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:13:58,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:13:58,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:13:58,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:13:58,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:00,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:01,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:01,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:01,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:01,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:01,913 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:01,914 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:01,919 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:02,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:03,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:04,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:06,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:06,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:06,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:06,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:07,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:08,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:09,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:10,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:14:10,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:10,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:10,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:10,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:11,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:12,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:14,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:16,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:16,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:17,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:17,007 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:17,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:17,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:17,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:18,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:20,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:21,814 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:21,815 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:21,815 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:21,820 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:22,447 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:24,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:25,821 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:26,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:26,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:26,266 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:26,271 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:26,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:28,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:30,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:30,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:30,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:30,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:31,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:31,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:31,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:31,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:31,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:32,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:35,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:35,492 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:35,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:35,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:35,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:36,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:36,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:40,157 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:14:40,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:40,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:40,512 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:40,517 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:40,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:42,518 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:42,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:46,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:14:46,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:14:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:14:48,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:48,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:48,507 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:48,507 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:48,513 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:48,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:50,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:53,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:14:53,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:14:53,693 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:14:53,693 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:14:53,699 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:14:54,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:14:54,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:14:58,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:01,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:01,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:02,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:02,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:02,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:02,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:02,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:04,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:04,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:07,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:07,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:07,202 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:07,207 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:07,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:08,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:10,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:15:10,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:12,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:12,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:12,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:12,419 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:12,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:14,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:15,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:16,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:16,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:17,445 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:17,446 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:17,446 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:17,452 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:17,557 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:18,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:20,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:24,521 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:24,522 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:24,522 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:24,527 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:24,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:25,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:26,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:29,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:29,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:29,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:29,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:30,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:30,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:30,773 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:31,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:31,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:31,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:34,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:34,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:34,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:34,833 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:35,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:35,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:36,773 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:39,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:39,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:39,004 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:39,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:39,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:40,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:15:40,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:41,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:44,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:44,158 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:44,159 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:44,163 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:44,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:46,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:15:46,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:15:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:15:46,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:46,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:48,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:48,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:48,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:48,172 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:48,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:50,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:15:52,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:56,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:15:56,147 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:15:56,147 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:15:56,153 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:15:56,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:15:58,154 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:15:58,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:00,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:00,679 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:00,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:00,684 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:00,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:01,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:01,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:01,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:02,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:03,315 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:05,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:05,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:05,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:05,070 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:05,806 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:06,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:09,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:09,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:09,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:09,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:09,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:09,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:10,164 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:16:10,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:14,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:15,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:15,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:15,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:15,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:15,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:16,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:16,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:16,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:19,359 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:20,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:20,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:20,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:20,397 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:20,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:22,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:24,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:24,837 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:24,837 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:24,838 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:24,843 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:25,826 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:26,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:29,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:29,212 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:29,213 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:29,218 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:29,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:30,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:30,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:31,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:31,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:33,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:33,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:33,748 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:33,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:33,836 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:34,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:35,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:40,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:16:41,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:42,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:42,096 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:42,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:42,102 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:42,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:44,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:46,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:16:46,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:16:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:16:46,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:48,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:48,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:48,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:48,444 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:48,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:50,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:51,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:55,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:55,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:55,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:55,206 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:55,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:16:56,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:16:57,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:16:59,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:16:59,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:16:59,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:16:59,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:16:59,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:00,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:01,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:01,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:02,345 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:03,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:03,954 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:03,954 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:03,959 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:03,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:06,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:07,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:10,168 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:17:12,122 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:12,123 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:12,123 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:12,128 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:12,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:13,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:14,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:16,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:16,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:17,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:17,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:17,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:17,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:17,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:18,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:20,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:23,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:23,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:23,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:23,977 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:23,982 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:24,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:26,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:28,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:28,516 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:28,517 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:28,522 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:29,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:29,523 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:31,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:31,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:31,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:35,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:37,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:37,420 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:37,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:37,425 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:38,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:39,031 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:40,170 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:17:41,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:45,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:45,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:45,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:45,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:46,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:46,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:17:46,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:17:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:17:46,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:47,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:49,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:49,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:49,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:49,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:50,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:51,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:51,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:54,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:54,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:54,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:54,056 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:55,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:17:57,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:17:57,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:17:58,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:17:58,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:17:58,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:17:58,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:17:59,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:01,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:01,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:01,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:02,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:03,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:03,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:03,135 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:03,141 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:04,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:05,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:08,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:09,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:09,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:09,831 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:09,836 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:10,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:10,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:18:11,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:14,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:14,125 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:14,125 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:14,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:14,130 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:15,064 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:16,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:16,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:17,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:19,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:22,911 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:22,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:22,912 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:22,916 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:23,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:24,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:25,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:27,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:27,200 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:27,200 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:27,205 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:28,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:29,079 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:30,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:31,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:31,057 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:31,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:31,707 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:31,707 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:31,712 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:32,081 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:33,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:35,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:36,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:36,306 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:36,306 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:36,311 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:37,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:39,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:40,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:18:40,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:40,930 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:40,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:40,931 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:40,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:41,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:43,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:45,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:46,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:18:46,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:18:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:18:48,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:48,236 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:48,236 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:48,242 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:49,095 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:51,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:51,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:18:52,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:18:52,679 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:18:52,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:18:52,684 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:18:53,100 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:18:55,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:18:56,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:01,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:01,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:01,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:01,458 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:01,458 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:01,464 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:02,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:02,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:03,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:06,990 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:06,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:06,991 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:06,996 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:07,115 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:07,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:09,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:10,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:19:11,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:11,463 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:11,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:11,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:12,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:13,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:13,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:16,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:16,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:16,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:16,661 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:16,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:16,666 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:17,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:18,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:19,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:20,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:20,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:20,766 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:20,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:21,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:23,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:23,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:27,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:27,068 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:27,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:27,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:27,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:29,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:29,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:31,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:31,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:34,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:35,497 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:35,498 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:35,498 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:35,503 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:36,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:37,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:39,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:40,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:19:44,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:44,325 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:44,325 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:44,330 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:45,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:45,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:46,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:19:46,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:19:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:19:47,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:49,514 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:49,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:49,515 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:49,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:50,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:50,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:51,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:19:54,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:19:54,983 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:19:54,983 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:19:54,988 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:19:55,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:19:55,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:19:57,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:00,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:00,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:00,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:00,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:01,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:01,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:01,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:01,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:03,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:05,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:05,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:05,046 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:05,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:05,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:06,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:07,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:09,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:09,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:09,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:09,721 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:10,184 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:20:10,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:11,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:11,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:15,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:15,987 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:15,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:15,993 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:16,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:16,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:16,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:16,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:17,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:20,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:20,722 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:20,722 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:20,727 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:21,199 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:21,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:23,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:25,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:25,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:25,145 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:25,150 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:25,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:27,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:27,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:30,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:30,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:30,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:30,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:31,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:31,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:31,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:32,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:33,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:35,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:35,088 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:35,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:35,093 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:35,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:37,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:38,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:39,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:39,473 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:39,473 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:39,478 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:40,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:20:40,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:41,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:43,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:46,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:20:46,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:20:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:20:46,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:46,362 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:46,362 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:46,368 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:47,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:48,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:49,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:50,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:50,777 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:50,777 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:50,782 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:51,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:20:53,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:20:53,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:58,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:20:59,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:20:59,080 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:20:59,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:20:59,085 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:20:59,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:01,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:01,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:01,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:03,617 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:03,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:03,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:03,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:04,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:04,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:05,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:07,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:07,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:07,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:07,798 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:08,247 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:09,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:09,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:10,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:21:11,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:11,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:11,899 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:11,904 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:12,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:13,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:14,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:16,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:16,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:16,402 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:16,402 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:16,402 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:16,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:17,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:19,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:20,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:22,522 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:22,523 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:22,523 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:22,528 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:23,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:25,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:25,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:27,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:27,489 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:27,489 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:27,494 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:28,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:29,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:31,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:31,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:31,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:33,752 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:33,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:33,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:33,768 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:34,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:35,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:36,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:38,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:38,239 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:38,239 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:38,244 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:38,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:40,187 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:21:41,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:42,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:43,611 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:43,612 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:43,612 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:43,617 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:44,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:45,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:46,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:21:46,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:21:46,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:21:47,343 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:47,886 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:47,887 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:47,887 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:47,892 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:48,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:49,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:52,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:21:53,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:21:53,761 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:21:53,761 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:21:53,766 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:21:54,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:21:55,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:21:58,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:00,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:00,018 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:00,019 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:00,024 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:00,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:01,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:01,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:01,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:01,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:04,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:09,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:09,569 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:09,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:09,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:09,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:10,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:22:10,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:11,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:13,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:13,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:13,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:13,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:14,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:14,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:15,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:16,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:16,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:16,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:19,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:19,418 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:19,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:19,424 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:20,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:20,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:21,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:24,373 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:24,374 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:24,374 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:24,380 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:25,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:26,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:27,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:28,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:28,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:28,860 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:28,865 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:29,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:31,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:31,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:31,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:31,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:32,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:33,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:33,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:33,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:33,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:33,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:35,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:38,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:39,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:39,270 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:39,270 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:39,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:39,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:40,194 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:22:41,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:43,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:44,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:44,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:44,414 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:44,419 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:45,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:46,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:22:46,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:22:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:22:47,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:48,311 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:49,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:49,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:49,889 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:49,894 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:50,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:51,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:53,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:54,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:54,107 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:54,107 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:54,112 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:54,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:22:55,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:22:59,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:22:59,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:22:59,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:22:59,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:22:59,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:22:59,371 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:01,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:01,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:01,410 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:04,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:06,189 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:06,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:06,190 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:06,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:06,413 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:07,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:10,193 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:23:10,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:10,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:10,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:10,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:10,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:11,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:13,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:15,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:16,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:16,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:16,270 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:16,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:16,315 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:16,320 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:16,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:17,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:21,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:25,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:25,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:25,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:25,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:25,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:27,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:27,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:31,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:31,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:32,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:33,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:33,592 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:33,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:33,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:34,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:35,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:37,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:38,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:38,900 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:38,901 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:38,906 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:39,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:40,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:23:41,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:43,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:45,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:45,103 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:45,103 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:45,109 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:45,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:46,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:23:46,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:23:46,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:23:47,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:48,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:50,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:50,196 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:50,196 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:50,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:50,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:51,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:54,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:54,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:54,733 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:54,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:54,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:55,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:23:57,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:23:59,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:23:59,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:23:59,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:23:59,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:23:59,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:23:59,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:01,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:01,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:01,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:05,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:06,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:06,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:06,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:06,055 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:06,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:07,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:10,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:24:11,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:11,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:11,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:11,635 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:11,641 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:12,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:13,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:16,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:16,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:16,199 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:16,486 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:16,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:16,487 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:16,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:16,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:19,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:21,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:21,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:21,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:21,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:21,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:22,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:23,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:25,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:25,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:25,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:25,648 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:26,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:27,529 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:27,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:31,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:31,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:31,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:31,819 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:31,819 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:31,824 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:32,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:32,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:33,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:37,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:39,465 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:39,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:39,466 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:39,471 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:39,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:40,203 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:24:41,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:43,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:44,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:44,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:44,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:44,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:44,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:45,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:46,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:24:46,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:24:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:24:48,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:49,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:49,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:49,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:49,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:50,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:51,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:53,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:54,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:54,551 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:54,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:54,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:54,589 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:24:57,592 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:24:59,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:24:59,551 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:24:59,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:24:59,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:24:59,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:24:59,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:01,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:01,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:01,597 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:03,819 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:03,820 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:03,820 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:03,825 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:04,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:04,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:05,601 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:09,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:09,521 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:09,521 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:09,526 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:09,603 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:10,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:25:10,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:11,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:14,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:14,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:14,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:14,240 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:14,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:15,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:15,612 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:16,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:16,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:18,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:18,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:18,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:18,855 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:19,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:20,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:21,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:25,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:25,076 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:25,076 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:25,081 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:25,620 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:26,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:27,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:31,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:31,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:31,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:32,765 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:32,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:32,766 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:32,771 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:33,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:35,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:36,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:37,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:37,500 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:37,500 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:37,505 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:37,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:39,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:40,204 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:25:42,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:44,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:44,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:44,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:44,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:44,650 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:45,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:46,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:25:46,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:25:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:25:47,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:50,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:50,171 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:50,171 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:50,176 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:50,655 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:51,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:53,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:25:55,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:25:55,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:25:55,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:25:55,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:25:56,661 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:25:57,663 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:25:58,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:01,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:01,060 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:04,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:04,130 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:04,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:04,130 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:04,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:04,667 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:05,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:09,175 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:09,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:09,186 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:09,187 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:09,192 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:09,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:10,206 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:26:11,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:13,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:13,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:13,489 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:13,494 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:13,677 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:14,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:15,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:16,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:16,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:18,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:18,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:18,542 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:18,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:18,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:19,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:21,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:24,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:24,139 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:24,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:24,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:24,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:25,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:25,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:29,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:29,275 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:29,275 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:29,281 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:29,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:30,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:31,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:31,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:31,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:33,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:33,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:33,532 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:33,537 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:33,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:35,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:35,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:37,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:37,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:37,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:37,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:37,703 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:39,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:40,208 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:26:41,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:42,501 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:42,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:42,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:42,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:42,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:43,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:46,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:26:46,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:26:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:26:46,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:46,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:46,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:46,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:46,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:47,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:49,750 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:51,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:51,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:51,647 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:51,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:51,652 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:51,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:53,755 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:26:56,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:26:58,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:26:58,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:26:58,411 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:26:58,416 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:26:58,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:26:59,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:01,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:01,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:02,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:03,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:03,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:03,818 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:03,823 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:04,763 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:05,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:07,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:08,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:08,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:08,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:08,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:08,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:10,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:27:11,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:13,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:14,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:14,020 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:14,020 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:14,025 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:14,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:15,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:16,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:16,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:18,326 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:19,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:19,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:19,685 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:19,691 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:19,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:20,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:23,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:24,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:24,956 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:24,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:24,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:25,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:26,811 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:28,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:31,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:31,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:31,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:31,567 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:31,568 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:31,572 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:31,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:32,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:34,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:39,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:40,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:40,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:40,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:40,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:40,213 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:27:40,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:40,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:44,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:44,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:44,528 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:44,533 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:44,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:44,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:45,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:46,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:27:46,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:27:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:27:48,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:48,809 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:48,809 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:48,814 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:48,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:27:50,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:50,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:55,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:27:56,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:27:56,323 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:27:56,324 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:27:56,329 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:27:56,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:27:56,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:01,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:01,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:01,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:01,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:01,732 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:01,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:01,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:01,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:02,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:06,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:06,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:06,366 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:06,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:06,371 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:06,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:06,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:10,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:28:11,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:11,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:11,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:11,066 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:11,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:12,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:12,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:15,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:15,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:15,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:15,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:15,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:16,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:16,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:16,863 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:17,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:19,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:19,934 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:19,935 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:19,940 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:20,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:20,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:22,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:24,301 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:24,302 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:24,302 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:24,307 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:24,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:24,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:28,308 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:30,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:30,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:30,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:30,247 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:30,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:30,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:31,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:31,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:33,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:35,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:35,975 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:35,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:35,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:36,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:36,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:39,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:40,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:28:41,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:41,604 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:41,605 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:41,610 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:41,894 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:42,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:44,611 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:46,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:28:46,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:28:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:28:46,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:46,415 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:46,415 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:46,420 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:46,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:46,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:50,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:51,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:51,153 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:51,153 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:51,158 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:51,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:52,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:28:55,569 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:28:55,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:28:55,570 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:28:55,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:28:55,575 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:28:55,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:28:56,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:00,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:01,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:01,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:01,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:01,294 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:01,294 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:01,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:01,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:02,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:05,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:05,262 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:05,263 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:05,268 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:05,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:06,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:06,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:10,219 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:29:12,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:13,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:13,790 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:13,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:13,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:13,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:14,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:16,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:16,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:17,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:18,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:18,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:18,470 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:18,475 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:18,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:18,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:22,515 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:22,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:22,699 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:22,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:22,704 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:22,943 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:22,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:27,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:27,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:27,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:27,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:27,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:27,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:28,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:31,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:31,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:32,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:32,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:32,341 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:32,346 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:32,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:32,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:33,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:36,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:36,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:36,637 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:36,642 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:36,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:36,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:38,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:40,221 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:29:41,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:41,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:41,868 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:41,873 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:41,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:42,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:43,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:46,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:29:46,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:29:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:29:47,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:47,583 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:47,583 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:47,588 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:47,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:48,970 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:49,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:54,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:29:54,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:54,713 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:54,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:54,719 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:54,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:29:54,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:29:59,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:29:59,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:29:59,198 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:29:59,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:29:59,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:00,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:00,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:01,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:01,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:05,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:05,114 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:05,115 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:05,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:05,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:06,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:06,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:09,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:09,625 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:09,626 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:09,631 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:09,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:10,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:30:11,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:11,225 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:14,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:14,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:14,013 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:14,018 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:15,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:15,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:16,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:16,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:16,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:18,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:18,393 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:18,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:18,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:19,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:19,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:22,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:23,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:23,706 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:23,706 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:23,711 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:24,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:25,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:27,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:30,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:30,071 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:30,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:30,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:31,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:31,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:31,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:31,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:33,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:35,765 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:35,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:35,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:35,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:36,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:37,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:38,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:40,225 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:30:42,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:42,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:42,695 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:42,700 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:43,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:43,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:44,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:46,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:30:46,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:30:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:30:48,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:48,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:48,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:48,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:49,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:49,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:50,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:54,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:54,549 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:54,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:54,554 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:30:55,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:30:55,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:30:55,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:30:59,225 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:30:59,226 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:30:59,226 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:30:59,231 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:00,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:01,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:01,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:01,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:01,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:03,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:03,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:03,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:03,622 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:04,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:05,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:06,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:07,806 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:07,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:07,807 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:07,812 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:08,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:09,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:10,227 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:31:12,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:16,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:16,062 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:16,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:16,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:16,265 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:16,271 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:17,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:17,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:17,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:20,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:20,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:20,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:20,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:21,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:21,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:22,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:25,962 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:25,963 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:25,963 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:25,968 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:26,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:27,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:27,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:31,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:31,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:31,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:32,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:32,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:32,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:32,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:33,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:33,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:33,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:37,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:37,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:37,721 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:37,726 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:38,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:38,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:39,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:40,233 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:31:44,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:44,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:44,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:44,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:44,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:45,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:45,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:46,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:31:46,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:31:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:31:49,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:50,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:50,080 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:50,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:50,085 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:50,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:51,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:31:55,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:31:58,037 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:31:58,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:31:58,038 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:31:58,043 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:31:58,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:31:59,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:01,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:01,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:01,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:02,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:02,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:02,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:02,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:03,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:03,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:06,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:08,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:08,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:08,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:08,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:09,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:09,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:10,232 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:32:12,233 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:13,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:13,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:13,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:13,071 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:13,144 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:15,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:16,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:16,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:16,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:17,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:21,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:21,567 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:21,567 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:21,572 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:22,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:22,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:23,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:25,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:25,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:25,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:25,774 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:26,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:27,157 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:27,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:30,204 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:30,205 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:30,205 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:30,210 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:31,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:31,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:31,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:31,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:31,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:33,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:35,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:35,867 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:35,867 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:35,872 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:36,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:37,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:38,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:40,237 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:32:40,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:40,770 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:40,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:40,777 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:41,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:41,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:44,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:45,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:45,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:45,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:45,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:46,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:32:46,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:32:46,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:32:46,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:47,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:50,079 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:50,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:50,080 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:50,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:50,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:50,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:51,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:54,601 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:54,602 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:54,602 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:54,607 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:55,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:32:55,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:32:55,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:32:59,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:32:59,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:32:59,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:32:59,105 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:32:59,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:01,062 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:01,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:01,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:01,103 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:01,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:03,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:03,460 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:03,460 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:03,465 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:04,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:05,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:06,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:07,965 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:07,966 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:07,966 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:07,971 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:08,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:09,204 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:10,236 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:33:12,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:12,425 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:12,425 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:12,426 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:12,431 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:13,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:13,216 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:16,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:16,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:17,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:20,581 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:20,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:20,582 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:20,586 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:21,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:21,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:22,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:24,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:24,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:24,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:24,855 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:25,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:25,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:27,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:29,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:29,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:29,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:29,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:30,232 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:31,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:31,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:31,234 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:33,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:35,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:35,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:35,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:35,421 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:36,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:37,242 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:38,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:39,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:39,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:39,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:39,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:40,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:33:40,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:41,246 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:44,240 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:44,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:44,860 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:44,860 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:44,865 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:45,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:45,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:46,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:33:46,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:33:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:33:48,878 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:48,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:48,879 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:48,884 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:49,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:49,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:49,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:33:54,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:33:54,614 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:33:54,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:33:54,620 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:33:55,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:33:55,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:33:55,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:00,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:01,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:01,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:02,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:02,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:02,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:02,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:03,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:03,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:06,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:09,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:09,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:09,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:09,695 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:10,240 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:34:10,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:11,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:11,275 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:14,506 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:14,507 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:14,508 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:14,513 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:15,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:15,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:16,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:16,063 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:16,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:19,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:19,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:19,600 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:19,605 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:20,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:21,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:21,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:26,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:26,088 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:26,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:26,093 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:26,287 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:27,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:27,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:30,631 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:30,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:30,632 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:30,637 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:31,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:31,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:31,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:31,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:32,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:34,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:34,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:34,638 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:34,644 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:35,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:35,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:37,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:40,242 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:34:42,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:42,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:42,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:42,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:42,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:43,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:43,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:46,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:34:46,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:34:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:34:46,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:46,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:46,928 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:46,933 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:47,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:47,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:47,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:52,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:34:52,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:34:52,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:34:52,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:34:53,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:34:53,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:34:53,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:34:58,819 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:01,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:01,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:01,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:01,373 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:01,373 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:01,379 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:02,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:03,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:04,379 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:06,866 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:06,867 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:06,867 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:06,872 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:07,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:07,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:09,873 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:10,247 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:35:11,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:11,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:11,331 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:11,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:11,366 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:13,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:15,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:16,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:16,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:16,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:16,422 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:16,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:16,427 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:17,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:17,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:20,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:24,046 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:24,047 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:24,047 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:24,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:24,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:25,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:26,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:31,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:31,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:31,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:32,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:32,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:32,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:32,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:32,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:33,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:36,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:38,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:38,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:38,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:38,721 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:39,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:39,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:40,246 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:35:42,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:43,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:43,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:43,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:43,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:43,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:43,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:46,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:35:46,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:35:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:35:47,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:50,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:50,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:50,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:50,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:51,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:51,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:52,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:55,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:55,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:55,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:55,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:35:56,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:35:57,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:35:58,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:35:59,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:35:59,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:35:59,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:35:59,619 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:00,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:01,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:01,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:01,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:04,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:05,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:05,743 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:05,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:05,748 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:06,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:07,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:09,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:10,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:36:12,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:12,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:12,231 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:12,236 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:12,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:13,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:15,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:16,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:16,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:18,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:18,939 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:18,939 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:18,944 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:19,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:19,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:20,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:23,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:23,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:23,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:23,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:24,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:25,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:26,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:31,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:31,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:32,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:32,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:32,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:32,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:32,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:32,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:33,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:37,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:40,251 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:36:40,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:40,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:40,960 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:40,965 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:41,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:41,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:42,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:46,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:36:46,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:36:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:36:46,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:46,641 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:46,642 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:46,647 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:47,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:47,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:48,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:51,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:51,511 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:51,511 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:51,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:52,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:36:53,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:54,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:36:57,042 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:36:57,043 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:36:57,043 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:36:57,048 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:36:57,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:36:57,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:00,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:01,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:01,064 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:02,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:02,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:02,042 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:02,047 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:02,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:03,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:05,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:09,090 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:09,091 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:09,091 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:09,096 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:09,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:09,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:10,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:10,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:37:13,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:13,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:13,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:13,278 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:13,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:13,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:15,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:16,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:16,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:20,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:21,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:21,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:21,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:21,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:22,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:23,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:25,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:27,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:27,662 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:27,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:27,668 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:28,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:29,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:31,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:31,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:31,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:32,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:32,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:32,500 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:32,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:32,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:33,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:36,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:36,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:36,549 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:36,549 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:36,554 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:36,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:37,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:40,255 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:37:40,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:40,808 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:40,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:40,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:41,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:41,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:41,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:46,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:37:46,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:37:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:37:47,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:47,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:47,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:47,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:47,622 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:48,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:49,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:52,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:54,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:54,317 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:54,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:54,322 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:54,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:37:55,582 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:58,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:37:59,054 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:37:59,055 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:37:59,055 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:37:59,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:37:59,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:37:59,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:01,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:01,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:03,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:05,465 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:05,465 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:05,466 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:05,471 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:05,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:07,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:08,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:10,257 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:38:11,807 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:11,808 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:11,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:11,814 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:12,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:13,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:13,814 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:16,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:16,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:18,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:18,936 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:18,937 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:18,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:18,942 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:19,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:19,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:23,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:24,178 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:24,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:24,179 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:24,184 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:24,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:25,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:28,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:28,388 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:28,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:28,394 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:28,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:29,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:29,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:31,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:31,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:35,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:36,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:36,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:36,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:36,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:37,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:37,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:40,263 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:38:41,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:43,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:43,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:43,339 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:43,344 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:43,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:43,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:46,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:38:46,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:38:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:38:46,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:48,842 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:48,843 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:48,843 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:48,848 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:49,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:49,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:51,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:54,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:54,085 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:54,085 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:54,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:54,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:38:55,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:57,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:38:58,933 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:38:58,933 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:38:58,934 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:38:58,939 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:38:59,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:38:59,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:01,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:01,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:02,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:03,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:03,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:03,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:03,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:04,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:05,718 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:08,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:08,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:08,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:08,329 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:08,334 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:08,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:09,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:10,261 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:39:13,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:15,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:15,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:15,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:15,561 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:15,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:16,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:16,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:17,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:18,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:21,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:21,026 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:21,026 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:21,031 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:21,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:21,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:24,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:25,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:25,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:25,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:25,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:26,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:27,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:29,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:30,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:30,480 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:30,481 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:30,486 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:30,743 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:31,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:31,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:31,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:35,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:35,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:35,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:35,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:35,952 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:36,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:37,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:40,266 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:39:40,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:40,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:40,828 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:40,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:40,833 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:41,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:41,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:45,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:46,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:39:46,065 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:39:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:39:46,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:46,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:46,506 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:46,511 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:46,795 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:47,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:51,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:52,198 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:52,199 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:52,199 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:52,204 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:52,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:39:53,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:57,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:39:57,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:39:57,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:39:57,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:39:57,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:39:57,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:39:57,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:01,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:01,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:01,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:01,954 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:01,955 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:01,959 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:02,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:02,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:03,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:07,996 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:08,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:08,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:08,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:08,675 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:08,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:09,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:10,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:40:13,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:13,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:13,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:13,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:13,030 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:13,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:13,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:16,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:16,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:18,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:18,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:18,425 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:18,425 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:18,430 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:18,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:19,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:22,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:22,816 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:22,816 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:22,821 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:22,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:23,821 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:23,832 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:27,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:27,800 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:27,801 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:27,806 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:27,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:29,806 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:29,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:31,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:31,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:33,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:33,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:33,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:33,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:33,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:33,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:35,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:38,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:38,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:38,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:38,287 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:38,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:39,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:40,267 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:40:41,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:42,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:42,812 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:42,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:42,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:42,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:43,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:46,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:40:46,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:40:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:40:46,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:48,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:48,247 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:48,247 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:48,252 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:48,859 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:49,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:52,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:54,123 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:54,124 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:54,125 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:54,130 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:54,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:40:55,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:58,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:40:58,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:40:58,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:40:58,977 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:40:58,982 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:40:59,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:40:59,872 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:01,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:01,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:03,314 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:05,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:05,088 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:05,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:05,093 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:05,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:05,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:09,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:09,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:09,512 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:09,513 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:09,517 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:09,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:09,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:10,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:41:14,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:14,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:14,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:14,071 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:14,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:15,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:15,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:16,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:16,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:18,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:18,537 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:18,537 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:18,542 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:18,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:19,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:20,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:25,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:25,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:25,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:25,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:25,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:25,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:27,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:30,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:31,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:31,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:31,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:31,297 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:31,302 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:31,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:31,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:36,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:36,026 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:36,026 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:36,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:36,032 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:36,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:37,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:40,272 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:41:40,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:40,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:40,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:40,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:40,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:41,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:41,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:46,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:41:46,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:41:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:41:46,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:46,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:46,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:46,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:46,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:47,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:47,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:51,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:51,534 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:51,534 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:51,539 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:51,946 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:51,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:52,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:41:57,315 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:41:57,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:41:57,316 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:41:57,321 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:41:57,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:41:57,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:41:58,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:01,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:01,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:01,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:02,001 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:02,001 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:02,006 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:02,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:03,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:04,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:06,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:06,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:06,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:06,682 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:06,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:07,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:09,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:10,273 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:42:11,509 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:11,510 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:11,510 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:11,515 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:11,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:11,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:15,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:16,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:16,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:16,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:16,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:16,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:16,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:16,972 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:17,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:20,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:20,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:20,581 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:20,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:20,587 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:20,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:21,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:24,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:24,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:24,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:24,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:24,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:25,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:25,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:29,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:29,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:29,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:29,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:29,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:29,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:31,065 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:31,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:31,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:34,051 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:34,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:34,051 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:34,057 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:34,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:35,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:36,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:39,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:39,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:39,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:39,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:40,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:40,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:40,276 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:42:41,277 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:44,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:44,051 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:44,052 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:44,057 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:45,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:46,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:46,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:42:46,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:42:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:42:46,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:49,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:49,452 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:49,452 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:49,457 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:50,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:42:50,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:51,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:55,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:42:55,714 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:42:55,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:42:55,720 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:42:56,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:42:56,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:42:58,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:00,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:00,272 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:00,272 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:00,277 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:01,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:01,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:01,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:02,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:02,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:05,242 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:05,243 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:05,243 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:05,248 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:06,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:06,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:08,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:09,877 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:09,878 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:09,878 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:09,884 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:10,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:10,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:43:12,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:14,283 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:14,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:14,416 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:14,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:14,422 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:15,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:16,050 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:16,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:16,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:18,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:18,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:18,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:18,526 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:19,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:19,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:20,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:24,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:24,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:24,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:24,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:25,058 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:25,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:26,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:30,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:31,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:31,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:31,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:32,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:32,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:32,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:32,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:33,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:34,068 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:35,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:37,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:37,350 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:37,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:37,355 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:38,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:38,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:40,280 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:43:41,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:41,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:41,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:41,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:41,472 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:42,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:42,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:46,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:43:46,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:43:46,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:43:46,373 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:50,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:50,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:50,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:50,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:51,083 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:51,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:52,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:43:56,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:43:56,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:43:56,174 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:43:56,179 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:43:57,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:43:57,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:43:58,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:00,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:00,075 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:00,076 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:00,081 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:00,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:01,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:01,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:01,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:02,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:02,357 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:05,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:05,021 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:05,021 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:05,026 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:05,098 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:06,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:08,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:09,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:09,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:09,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:09,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:10,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:10,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:10,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:44:13,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:13,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:13,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:13,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:13,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:14,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:14,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:16,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:16,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:16,107 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:18,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:21,814 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:21,814 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:21,815 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:21,820 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:22,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:23,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:24,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:27,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:27,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:27,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:27,020 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:27,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:28,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:29,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:31,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:31,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:32,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:32,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:32,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:32,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:33,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:34,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:34,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:37,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:37,285 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:37,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:37,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:38,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:38,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:40,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:44:40,286 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:42,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:42,408 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:42,408 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:42,413 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:43,164 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:44,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:45,414 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:46,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:44:46,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:44:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:44:47,081 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:47,082 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:47,082 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:47,088 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:47,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:48,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:51,124 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:51,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:51,381 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:51,382 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:51,387 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:52,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:44:52,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:56,057 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:44:56,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:44:56,058 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:44:56,063 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:44:56,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:44:57,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:44:58,193 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:01,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:01,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:01,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:01,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:01,438 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:01,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:01,443 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:02,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:02,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:02,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:05,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:05,716 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:05,716 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:05,721 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:06,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:06,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:07,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:09,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:09,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:09,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:09,920 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:10,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:10,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:45:12,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:13,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:14,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:14,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:14,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:14,400 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:15,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:16,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:16,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:16,213 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:19,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:19,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:19,003 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:19,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:19,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:19,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:20,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:24,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:25,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:25,355 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:25,355 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:25,360 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:26,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:26,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:29,361 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:30,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:30,876 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:30,876 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:30,881 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:31,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:31,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:31,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:32,264 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:35,326 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:35,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:35,860 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:35,860 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:35,865 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:36,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:38,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:40,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:40,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:40,085 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:40,089 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:40,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:40,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:45:41,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:42,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:44,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:44,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:44,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:44,916 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:45,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:46,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:45:46,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:45:46,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:45:46,279 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:47,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:49,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:49,441 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:49,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:49,446 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:50,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:45:52,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:45:52,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:57,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:45:57,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:45:57,792 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:45:57,792 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:45:57,797 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:45:58,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:00,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:01,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:01,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:03,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:03,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:03,959 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:03,959 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:03,964 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:04,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:06,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:08,845 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:08,846 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:08,846 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:08,846 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:08,851 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:09,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:10,291 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:46:10,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:14,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:15,359 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:15,359 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:15,359 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:15,365 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:16,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:16,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:16,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:16,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:19,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:20,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:20,452 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:20,452 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:20,456 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:21,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:22,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:24,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:28,527 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:28,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:28,528 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:28,532 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:29,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:29,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:30,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:31,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:31,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:33,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:33,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:33,028 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:33,033 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:33,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:34,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:35,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:39,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:39,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:39,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:39,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:39,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:40,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:40,293 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:46:40,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:45,294 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:46,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:46:46,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:46:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:46:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:46,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:46,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:46,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:47,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:48,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:50,776 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:51,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:51,437 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:51,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:51,442 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:52,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:46:52,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:46:56,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:46:58,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:46:58,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:46:58,203 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:46:58,208 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:46:58,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:00,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:01,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:01,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:02,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:03,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:03,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:03,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:03,324 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:03,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:04,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:07,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:08,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:08,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:08,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:08,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:08,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:10,295 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:47:10,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:13,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:13,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:13,888 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:13,889 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:13,894 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:14,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:16,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:16,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:16,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:18,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:18,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:18,972 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:18,972 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:18,977 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:19,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:20,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:23,051 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:23,052 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:23,052 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:23,061 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:23,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:24,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:24,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:27,607 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:27,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:27,608 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:27,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:28,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:29,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:30,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:31,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:31,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:34,175 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:34,176 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:34,176 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:34,180 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:34,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:35,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:36,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:38,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:38,681 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:38,682 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:38,686 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:39,434 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:40,301 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:47:40,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:40,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:43,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:43,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:43,940 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:43,945 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:44,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:45,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:46,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:47:46,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:47:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:47:46,442 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:48,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:48,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:48,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:48,453 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:49,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:50,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:51,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:53,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:53,427 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:53,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:53,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:53,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:47:54,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:47:57,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:47:58,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:47:58,538 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:47:58,538 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:47:58,543 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:47:59,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:00,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:01,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:01,068 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:03,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:03,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:03,585 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:03,585 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:03,590 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:04,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:06,471 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:08,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:10,299 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:48:11,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:11,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:11,884 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:11,889 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:12,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:13,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:14,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:16,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:16,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:16,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:16,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:16,689 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:16,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:16,695 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:17,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:18,483 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:19,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:22,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:22,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:22,444 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:22,450 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:22,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:24,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:25,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:26,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:26,927 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:26,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:26,932 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:27,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:28,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:30,932 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:31,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:31,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:34,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:34,323 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:34,323 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:34,328 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:34,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:36,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:36,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:38,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:38,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:38,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:38,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:38,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:40,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:48:40,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:42,306 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:43,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:43,072 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:43,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:43,077 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:43,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:44,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:46,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:48:46,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:48:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:48:48,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:48,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:48,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:48,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:48,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:48,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:50,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:53,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:54,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:54,630 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:54,631 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:54,636 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:55,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:48:56,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:48:58,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:48:59,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:48:59,070 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:48:59,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:48:59,075 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:48:59,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:00,532 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:01,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:01,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:03,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:03,509 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:03,509 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:03,514 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:03,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:04,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:04,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:08,497 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:08,498 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:08,498 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:08,504 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:08,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:10,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:49:10,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:10,542 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:13,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:13,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:13,572 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:13,577 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:14,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:14,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:15,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:16,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:16,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:18,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:18,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:18,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:18,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:18,551 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:20,557 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:21,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:23,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:23,662 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:23,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:23,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:24,562 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:26,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:26,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:27,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:27,905 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:27,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:27,910 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:28,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:30,574 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:31,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:31,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:32,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:32,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:32,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:32,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:32,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:32,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:34,579 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:36,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:36,965 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:36,965 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:36,970 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:37,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:37,971 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:38,586 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:40,309 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:49:41,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:41,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:41,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:41,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:42,591 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:43,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:44,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:46,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:49:46,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:49:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:49:46,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:46,333 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:46,333 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:46,338 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:46,595 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:48,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:49,339 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:51,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:51,892 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:51,892 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:51,898 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:52,613 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:54,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:49:54,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:49:56,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:49:56,186 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:49:56,187 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:49:56,192 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:49:56,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:49:58,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:00,193 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:01,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:01,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:04,762 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:04,762 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:04,762 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:04,767 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:05,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:05,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:06,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:08,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:08,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:08,954 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:08,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:09,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:10,308 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:50:10,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:11,309 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:13,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:13,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:13,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:13,603 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:13,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:14,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:16,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:16,069 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:17,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:18,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:18,020 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:18,020 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:18,025 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:18,652 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:20,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:23,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:23,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:23,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:23,041 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:23,046 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:23,662 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:24,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:27,139 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:27,140 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:27,140 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:27,145 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:27,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:28,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:28,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:31,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:31,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:32,827 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:32,827 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:32,827 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:32,833 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:33,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:33,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:34,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:38,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:38,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:38,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:38,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:38,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:39,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:40,310 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:50:40,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:42,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:42,407 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:42,407 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:42,412 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:42,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:44,413 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:44,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:46,069 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:50:46,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:50:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:50:47,921 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:47,921 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:47,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:47,926 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:48,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:49,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:50,740 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:52,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:52,619 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:52,620 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:52,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:52,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:54,745 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:50:55,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:50:57,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:50:57,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:50:57,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:50:57,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:50:57,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:50:58,749 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:01,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:01,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:01,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:02,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:02,457 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:02,457 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:02,462 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:02,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:04,778 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:06,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:07,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:07,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:07,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:07,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:07,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:08,782 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:10,312 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:51:11,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:11,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:11,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:11,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:11,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:12,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:14,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:16,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:16,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:17,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:20,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:20,377 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:20,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:20,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:20,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:22,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:22,796 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:27,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:30,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:30,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:30,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:30,780 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:30,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:31,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:31,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:32,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:33,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:36,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:36,247 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:36,247 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:36,252 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:36,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:38,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:39,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:40,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:51:42,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:42,013 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:42,013 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:42,018 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:42,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:44,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:45,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:46,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:51:46,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:51:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:51:47,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:47,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:47,231 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:47,236 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:47,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:48,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:50,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:52,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:52,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:52,135 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:52,140 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:52,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:54,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:51:56,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:51:57,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:51:57,820 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:51:57,820 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:51:57,825 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:51:57,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:51:58,865 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:01,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:01,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:01,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:02,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:02,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:02,268 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:02,273 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:02,867 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:04,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:06,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:06,676 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:06,676 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:06,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:06,681 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:06,871 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:08,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:10,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:52:10,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:10,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:10,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:10,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:11,876 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:11,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:12,880 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:15,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:15,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:15,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:15,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:15,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:16,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:16,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:16,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:17,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:20,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:20,358 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:20,358 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:20,363 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:20,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:22,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:22,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:24,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:24,629 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:24,629 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:24,634 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:24,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:26,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:27,635 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:31,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:31,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:31,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:31,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:31,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:31,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:31,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:32,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:32,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:36,909 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:36,910 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:36,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:36,915 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:37,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:37,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:38,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:40,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:52:43,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:43,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:43,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:43,429 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:43,434 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:43,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:44,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:46,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:52:46,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:52:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:52:48,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:52,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:52,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:52,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:52,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:52,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:54,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:52:54,925 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:56,806 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:52:56,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:52:56,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:52:56,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:52:56,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:52:58,929 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:52:59,813 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:01,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:01,070 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:05,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:05,138 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:05,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:05,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:05,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:05,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:06,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:09,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:09,393 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:09,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:09,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:09,943 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:10,323 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:53:10,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:10,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:14,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:14,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:14,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:14,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:14,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:15,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:16,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:16,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:16,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:19,313 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:19,314 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:19,314 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:19,319 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:19,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:20,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:21,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:24,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:24,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:24,174 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:24,185 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:24,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:26,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:27,185 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:31,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:31,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:31,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:31,603 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:31,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:31,608 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:31,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:32,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:32,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:37,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:37,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:37,470 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:37,475 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:37,992 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:38,475 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:38,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:40,322 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:53:42,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:42,860 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:42,861 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:42,866 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:42,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:43,866 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:45,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:46,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:53:46,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:53:46,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:53:49,246 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:49,246 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:49,247 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:49,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:49,252 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:50,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:51,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:54,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:55,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:55,034 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:55,034 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:55,039 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:53:56,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:53:57,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:53:59,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:53:59,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:53:59,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:53:59,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:53:59,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:00,031 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:01,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:01,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:01,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:01,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:05,318 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:05,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:05,360 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:05,360 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:05,365 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:06,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:07,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:09,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:09,749 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:09,749 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:09,754 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:10,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:10,327 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:54:10,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:11,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:14,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:14,377 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:14,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:14,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:15,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:15,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:16,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:16,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:16,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:17,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:19,418 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:19,419 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:19,419 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:19,424 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:20,066 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:20,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:21,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:23,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:23,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:23,592 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:23,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:24,075 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:25,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:25,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:29,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:29,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:29,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:29,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:30,085 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:31,070 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:31,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:31,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:31,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:31,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:34,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:34,912 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:34,913 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:34,918 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:35,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:36,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:37,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:39,369 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:39,370 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:39,370 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:39,375 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:40,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:40,327 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:54:41,104 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:42,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:45,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:45,254 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:45,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:45,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:46,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:54:46,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:54:46,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:54:47,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:47,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:49,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:49,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:49,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:49,662 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:50,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:54:51,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:54:52,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:57,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:54:58,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:54:58,502 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:54:58,502 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:54:58,507 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:54:59,121 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:01,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:01,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:01,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:03,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:04,314 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:04,315 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:04,315 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:04,320 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:05,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:07,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:08,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:10,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:55:12,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:12,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:12,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:12,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:13,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:13,494 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:15,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:16,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:16,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:16,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:17,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:17,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:17,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:17,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:19,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:19,140 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:22,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:22,093 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:22,093 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:22,098 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:22,142 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:24,099 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:25,145 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:26,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:26,743 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:26,743 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:26,748 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:27,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:29,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:29,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:31,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:31,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:33,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:33,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:33,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:33,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:33,153 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:35,088 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:35,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:37,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:37,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:37,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:37,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:38,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:39,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:40,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:55:40,333 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:44,787 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:44,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:44,788 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:44,793 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:45,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:45,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:46,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:55:46,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:55:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:55:47,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:49,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:49,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:49,609 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:49,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:50,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:51,173 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:51,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:54,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:54,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:54,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:54,397 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:55,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:55:57,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:55:57,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:55:59,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:55:59,036 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:55:59,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:55:59,041 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:55:59,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:01,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:01,071 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:01,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:03,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:03,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:03,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:03,939 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:03,943 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:04,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:05,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:08,730 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:08,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:08,731 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:08,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:08,736 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:09,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:10,333 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:56:11,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:14,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:14,006 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:14,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:14,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:14,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:14,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:15,199 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:16,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:16,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:18,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:18,108 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:18,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:18,114 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:18,201 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:19,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:21,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:24,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:27,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:27,122 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:27,122 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:27,127 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:27,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:29,214 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:30,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:31,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:31,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:32,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:32,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:32,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:32,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:32,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:35,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:35,236 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:36,867 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:36,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:36,868 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:36,874 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:37,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:39,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:40,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:56:40,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:42,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:42,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:42,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:42,922 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:43,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:45,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:45,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:46,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:56:46,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:56:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:56:47,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:47,535 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:47,535 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:47,540 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:48,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:49,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:51,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:52,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:52,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:52,385 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:52,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:53,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:55,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:56:56,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:56:56,679 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:56:56,680 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:56:56,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:56:56,685 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:56:57,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:56:59,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:01,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:01,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:01,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:01,666 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:01,667 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:01,671 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:01,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:02,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:03,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:06,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:07,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:07,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:07,518 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:07,523 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:08,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:09,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:10,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:57:12,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:15,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:15,286 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:15,286 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:15,292 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:15,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:16,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:16,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:17,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:17,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:22,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:23,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:23,705 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:23,705 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:23,710 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:24,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:25,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:27,748 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:27,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:27,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:27,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:27,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:28,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:29,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:31,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:31,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:32,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:32,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:32,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:32,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:33,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:33,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:35,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:37,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:37,453 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:37,453 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:37,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:38,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:39,323 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:39,459 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:40,342 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:57:43,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:43,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:43,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:43,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:43,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:45,147 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:45,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:46,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:57:46,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:57:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:57:47,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:47,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:47,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:47,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:48,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:49,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:57:50,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:55,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:57:56,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:57:56,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:57:56,238 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:57:56,243 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:57:56,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:57:57,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:00,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:00,893 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:00,893 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:00,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:00,898 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:01,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:01,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:01,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:03,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:05,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:05,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:05,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:05,399 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:06,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:06,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:07,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:10,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:58:12,343 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:13,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:13,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:13,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:13,877 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:14,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:15,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:16,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:16,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:17,361 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:19,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:19,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:19,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:19,742 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:20,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:21,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:22,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:25,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:25,189 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:25,189 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:25,194 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:25,411 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:27,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:28,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:30,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:30,597 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:30,597 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:30,602 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:31,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:31,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:31,418 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:33,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:33,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:36,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:36,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:36,341 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:36,346 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:36,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:38,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:39,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:40,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:58:41,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:41,721 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:41,721 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:41,726 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:42,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:43,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:43,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:46,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:58:46,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:58:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:58:49,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:50,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:50,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:50,690 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:50,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:51,436 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:53,439 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:54,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:58:55,920 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:58:55,921 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:58:55,921 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:58:55,926 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:58:56,441 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:58:57,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:58:59,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:00,979 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:00,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:00,980 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:00,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:01,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:01,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:01,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:03,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:05,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:06,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:06,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:06,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:06,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:07,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:09,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:10,345 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:59:11,291 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:11,292 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:11,292 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:11,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:11,297 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:11,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:13,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:15,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:15,506 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:15,506 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:15,511 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:16,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:16,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:16,293 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:16,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:17,464 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:21,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:21,235 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:21,235 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:21,241 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:21,467 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:22,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:23,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:25,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:25,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:25,445 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:25,450 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:25,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:27,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:27,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:30,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:30,926 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:30,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:30,932 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:31,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:31,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:31,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:33,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:33,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:36,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:36,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:36,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:36,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:37,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:38,720 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:39,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:40,347 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 17:59:41,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:41,661 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:41,661 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:41,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:42,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:43,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:44,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:46,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 17:59:46,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 17:59:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 17:59:47,712 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:47,713 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:47,713 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:47,718 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:48,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:49,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:49,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:52,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:52,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:52,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:53,009 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 17:59:53,497 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 17:59:55,010 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 17:59:55,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 17:59:59,506 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 17:59:59,507 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 17:59:59,507 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 17:59:59,512 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:00,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:00,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:01,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:01,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:01,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:04,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:04,002 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:04,002 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:04,007 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:04,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:05,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:06,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:10,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:10,094 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:10,094 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:10,099 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:10,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:00:10,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:11,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:11,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:15,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:15,215 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:15,215 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:15,220 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:15,518 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:16,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:16,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:17,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:17,522 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:21,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:21,978 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:21,978 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:21,983 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:22,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:23,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:23,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:26,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:26,652 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:26,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:26,657 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:27,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:28,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:29,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:31,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:31,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:31,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:31,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:31,469 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:31,474 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:31,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:33,539 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:34,512 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:35,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:35,858 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:35,858 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:35,864 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:36,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:37,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:39,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:40,351 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:00:41,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:41,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:41,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:41,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:41,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:43,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:45,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:45,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:45,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:45,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:45,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:46,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:00:46,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:00:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:00:46,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:47,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:50,311 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:50,311 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:50,312 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:50,312 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:50,317 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:50,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:51,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:54,525 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:54,526 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:54,526 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:54,531 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:00:54,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:00:55,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:00:57,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:00:59,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:00:59,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:00:59,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:00:59,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:00,567 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:00,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:01,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:01,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:01,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:05,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:05,828 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:05,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:05,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:05,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:06,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:07,575 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:10,357 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:01:11,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:13,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:13,134 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:13,135 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:13,140 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:13,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:15,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:16,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:16,073 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:17,348 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:17,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:17,757 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:17,757 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:17,762 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:18,585 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:19,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:22,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:23,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:23,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:23,325 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:23,330 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:23,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:25,637 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:28,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:29,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:29,503 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:29,503 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:29,508 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:29,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:31,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:31,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:31,643 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:33,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:34,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:34,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:34,166 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:34,171 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:34,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:35,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:39,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:40,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:01:44,279 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:44,279 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:44,280 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:44,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:44,285 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:44,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:45,656 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:46,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:01:46,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:01:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:01:48,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:48,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:48,146 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:48,151 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:48,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:49,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:50,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:53,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:53,598 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:53,599 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:53,604 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:53,692 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:55,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:01:55,695 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:01:58,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:01:58,099 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:01:58,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:01:58,104 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:01:58,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:01:59,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:01,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:01,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:01,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:02,871 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:02,872 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:02,872 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:02,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:03,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:05,705 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:06,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:06,943 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:06,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:06,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:06,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:07,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:09,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:10,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:02:11,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:11,809 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:11,809 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:11,814 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:12,712 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:12,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:13,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:16,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:16,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:16,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:16,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:16,376 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:16,381 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:16,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:17,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:18,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:23,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:23,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:23,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:23,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:23,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:24,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:25,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:27,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:27,350 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:27,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:27,355 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:27,728 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:29,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:29,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:31,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:31,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:31,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:31,858 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:31,858 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:31,864 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:32,734 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:33,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:34,900 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:36,327 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:36,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:36,328 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:36,333 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:36,738 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:37,741 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:40,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:40,360 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:02:41,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:41,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:41,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:41,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:41,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:43,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:45,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:46,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:02:46,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:02:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:02:46,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:46,355 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:46,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:46,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:46,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:47,760 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:50,953 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:50,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:50,954 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:50,954 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:50,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:51,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:53,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:02:55,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:02:55,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:02:55,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:02:55,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:02:55,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:02:56,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:02:57,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:00,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:00,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:00,469 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:00,475 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:00,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:01,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:01,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:01,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:02,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:04,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:04,960 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:04,961 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:04,966 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:05,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:07,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:08,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:09,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:09,433 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:09,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:09,439 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:09,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:10,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:03:11,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:13,367 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:16,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:16,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:17,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:17,007 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:17,007 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:17,012 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:17,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:19,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:19,798 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:22,518 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:22,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:22,519 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:22,524 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:22,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:23,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:24,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:27,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:27,016 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:27,017 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:27,022 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:27,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:29,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:30,022 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:31,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:31,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:34,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:34,179 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:34,179 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:34,184 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:34,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:35,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:35,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:40,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:40,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:03:40,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:40,529 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:40,529 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:40,534 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:40,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:41,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:44,837 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:44,837 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:44,838 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:44,843 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:45,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:45,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:46,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:03:46,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:03:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:03:47,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:49,915 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:49,915 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:49,915 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:49,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:50,902 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:50,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:51,904 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:54,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:54,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:54,178 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:54,183 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:54,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:03:55,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:03:56,183 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:03:59,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:03:59,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:03:59,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:03:59,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:03:59,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:01,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:01,074 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:01,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:01,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:04,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:04,277 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:04,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:04,282 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:04,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:05,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:07,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:08,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:08,646 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:08,646 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:08,651 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:08,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:09,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:10,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:04:12,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:16,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:16,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:17,181 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:17,182 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:17,182 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:17,187 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:17,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:18,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:19,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:22,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:22,590 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:22,591 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:22,596 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:22,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:23,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:23,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:27,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:27,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:27,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:27,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:27,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:28,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:29,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:31,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:31,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:31,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:33,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:33,377 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:33,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:33,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:33,949 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:34,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:35,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:39,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:39,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:39,464 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:39,464 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:39,469 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:39,955 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:40,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:04:41,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:43,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:43,924 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:43,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:43,929 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:43,960 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:44,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:45,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:46,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:04:46,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:04:46,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:04:50,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:50,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:50,434 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:50,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:50,439 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:50,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:51,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:04:55,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:04:56,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:04:56,497 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:04:56,497 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:04:56,502 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:04:57,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:04:58,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:00,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:01,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:01,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:01,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:01,791 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:01,791 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:01,791 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:01,796 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:02,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:04,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:05,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:07,521 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:07,522 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:07,522 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:07,528 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:08,011 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:10,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:10,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:05:11,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:13,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:13,288 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:13,288 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:13,293 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:14,017 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:16,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:16,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:16,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:16,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:16,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:19,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:19,808 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:19,808 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:19,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:20,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:21,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:22,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:24,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:24,488 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:24,488 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:24,493 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:25,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:26,045 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:27,494 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:29,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:29,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:29,291 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:29,296 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:30,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:31,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:31,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:31,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:32,052 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:33,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:36,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:36,214 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:36,214 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:36,219 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:37,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:38,058 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:39,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:40,318 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:40,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:40,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:40,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:40,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:05:41,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:42,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:44,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:45,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:45,412 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:45,412 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:45,426 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:46,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:46,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:05:46,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:05:46,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:05:48,068 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:49,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:49,836 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:49,836 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:49,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:49,841 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:50,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:52,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:54,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:54,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:54,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:54,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:05:55,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:05:55,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:05:55,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:05:59,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:05:59,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:05:59,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:05:59,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:00,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:00,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:01,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:01,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:01,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:01,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:06,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:08,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:08,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:08,584 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:08,589 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:09,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:09,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:10,375 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:06:11,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:15,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:15,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:15,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:15,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:16,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:16,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:16,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:16,202 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:17,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:17,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:21,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:21,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:21,505 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:21,510 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:22,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:22,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:23,210 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:26,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:26,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:26,135 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:26,141 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:26,212 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:27,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:28,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:31,075 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:31,075 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:31,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:31,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:31,593 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:31,593 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:31,598 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:32,250 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:33,253 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:33,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:38,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:40,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:40,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:40,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:40,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:40,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:40,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:06:41,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:44,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:45,510 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:45,510 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:45,511 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:45,516 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:46,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:06:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:06:46,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:06:47,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:49,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:50,763 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:50,764 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:50,764 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:50,769 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:51,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:06:51,270 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:54,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:06:56,215 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:06:56,216 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:06:56,216 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:06:56,221 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:06:56,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:06:57,276 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:00,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:01,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:01,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:01,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:04,805 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:04,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:04,806 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:04,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:05,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:05,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:05,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:10,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:07:11,381 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:12,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:12,328 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:12,329 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:12,334 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:13,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:13,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:16,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:16,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:16,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:16,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:16,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:16,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:16,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:16,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:17,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:17,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:21,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:21,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:21,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:21,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:21,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:22,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:23,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:26,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:27,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:27,280 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:27,280 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:27,285 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:27,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:29,311 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:31,386 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:31,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:31,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:31,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:31,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:31,729 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:31,729 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:31,734 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:32,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:33,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:36,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:36,373 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:36,373 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:36,378 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:37,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:37,321 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:37,379 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:40,381 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:07:40,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:40,931 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:40,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:40,936 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:41,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:41,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:42,936 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:45,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:45,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:45,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:45,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:46,328 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:07:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:07:46,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:07:47,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:48,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:50,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:50,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:50,121 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:50,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:50,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:51,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:54,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:07:54,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:54,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:54,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:54,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:55,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:55,340 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:58,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:07:58,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:07:58,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:07:58,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:07:59,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:07:59,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:07:59,929 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:01,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:01,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:01,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:03,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:03,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:03,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:03,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:03,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:05,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:05,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:07,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:07,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:07,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:07,453 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:08,351 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:09,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:10,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:08:10,384 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:12,802 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:12,803 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:12,803 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:12,808 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:13,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:13,358 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:15,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:16,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:16,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:16,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:18,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:18,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:18,964 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:18,969 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:19,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:21,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:21,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:24,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:24,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:24,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:24,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:25,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:25,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:26,442 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:29,162 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:29,163 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:29,163 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:29,168 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:29,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:31,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:31,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:31,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:31,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:31,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:34,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:34,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:34,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:34,255 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:34,406 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:35,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:37,292 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:40,385 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:08:40,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:40,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:40,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:40,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:41,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:41,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:42,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:46,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:46,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:46,246 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:46,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:08:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:08:46,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:08:46,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:47,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:48,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:51,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:51,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:51,129 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:51,134 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:51,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:53,425 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:54,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:08:56,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:08:56,186 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:08:56,187 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:08:56,192 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:08:56,477 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:08:57,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:08:59,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:01,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:01,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:01,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:04,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:04,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:04,728 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:04,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:04,733 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:05,486 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:05,487 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:09,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:10,391 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:09:11,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:11,576 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:11,576 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:11,581 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:12,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:13,493 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:15,582 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:15,794 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:15,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:15,795 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:15,801 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:16,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:16,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:16,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:17,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:20,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:22,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:22,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:22,608 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:22,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:23,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:23,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:26,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:27,926 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:27,926 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:27,927 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:27,932 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:28,508 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:29,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:31,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:31,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:31,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:32,610 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:33,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:33,225 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:33,225 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:33,230 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:33,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:35,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:38,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:40,390 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:09:43,078 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:43,079 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:43,079 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:43,084 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:43,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:43,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:44,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:46,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:09:46,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:09:46,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:09:48,521 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:48,522 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:48,522 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:48,527 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:48,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:49,530 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:49,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:53,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:09:53,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:09:53,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:09:53,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:09:54,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:09:54,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:09:55,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:09:59,810 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:00,160 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:00,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:00,161 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:00,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:00,544 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:01,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:01,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:01,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:01,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:05,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:05,667 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:05,668 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:05,668 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:05,673 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:06,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:07,551 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:10,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:10,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:10,366 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:10,371 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:10,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:10:10,553 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:11,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:11,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:16,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:16,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:16,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:16,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:16,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:16,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:17,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:17,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:21,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:22,334 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:22,334 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:22,334 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:22,339 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:22,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:23,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:26,999 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:27,000 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:27,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:27,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:27,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:27,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:27,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:31,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:31,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:31,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:31,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:31,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:31,675 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:31,680 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:32,577 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:32,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:33,579 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:36,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:36,589 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:36,590 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:36,597 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:37,585 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:37,585 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:38,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:40,394 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:10:42,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:42,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:42,338 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:42,343 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:42,606 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:43,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:44,344 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:46,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:10:46,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:10:46,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:10:48,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:48,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:48,643 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:48,648 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:48,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:49,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:49,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:52,883 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:52,884 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:52,884 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:52,889 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:53,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:10:53,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:54,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:10:57,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:10:57,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:10:57,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:10:57,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:10:57,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:10:59,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:00,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:01,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:01,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:01,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:01,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:01,925 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:01,925 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:01,930 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:02,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:03,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:05,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:06,356 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:06,357 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:06,357 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:06,362 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:06,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:07,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:10,396 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:11:10,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:10,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:10,835 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:10,840 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:11,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:11,759 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:11,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:16,225 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:16,225 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:16,225 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:16,231 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:16,388 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:16,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:17,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:17,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:20,607 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:20,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:20,608 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:20,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:20,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:21,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:22,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:24,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:24,980 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:24,981 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:24,985 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:25,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:25,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:27,986 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:30,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:30,897 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:30,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:30,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:30,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:31,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:31,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:31,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:31,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:33,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:38,621 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:39,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:39,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:39,322 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:39,326 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:39,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:40,398 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:11:41,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:44,377 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:44,378 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:44,378 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:44,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:44,383 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:44,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:45,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:46,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:11:46,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:11:46,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:11:49,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:49,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:49,218 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:49,223 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:49,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:50,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:51,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:11:55,260 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:11:55,595 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:11:55,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:11:55,596 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:11:55,601 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:11:55,937 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:11:57,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:00,481 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:00,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:00,482 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:00,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:00,487 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:00,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:01,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:01,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:01,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:01,944 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:05,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:06,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:06,708 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:06,708 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:06,713 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:06,947 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:07,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:10,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:12:11,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:13,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:13,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:13,493 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:13,498 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:13,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:15,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:16,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:16,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:18,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:18,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:18,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:18,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:18,959 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:19,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:21,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:23,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:23,282 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:23,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:23,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:23,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:25,974 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:27,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:27,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:27,745 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:27,745 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:27,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:27,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:29,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:31,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:31,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:32,610 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:33,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:33,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:33,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:33,659 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:33,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:35,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:37,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:39,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:39,805 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:39,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:39,811 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:39,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:40,402 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:12:41,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:43,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:46,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:12:46,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:12:46,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:12:47,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:47,151 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:47,151 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:47,156 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:48,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:48,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:49,192 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:52,223 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:52,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:52,224 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:52,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:53,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:54,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:54,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:12:56,623 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:12:56,624 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:12:56,624 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:12:56,629 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:12:57,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:12:58,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:12:59,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:00,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:00,831 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:00,831 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:00,836 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:01,057 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:01,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:01,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:01,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:02,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:04,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:05,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:05,424 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:05,424 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:05,430 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:06,103 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:08,107 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:10,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:10,096 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:10,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:10,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:10,101 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:10,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:10,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:13:12,111 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:14,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:14,641 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:14,642 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:14,647 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:15,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:15,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:16,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:16,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:16,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:20,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:20,176 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:20,177 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:20,182 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:21,119 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:21,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:22,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:24,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:24,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:24,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:24,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:25,123 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:26,126 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:26,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:28,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:28,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:28,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:28,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:29,128 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:30,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:31,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:31,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:33,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:33,232 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:33,232 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:33,238 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:34,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:34,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:37,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:40,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:40,027 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:40,028 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:40,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:40,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:40,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:13:42,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:42,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:45,147 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:45,148 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:45,148 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:45,153 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:45,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:46,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:46,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:13:46,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:13:46,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:13:47,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:49,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:49,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:49,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:49,738 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:50,191 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:52,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:52,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:13:54,155 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:13:54,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:13:54,157 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:13:54,162 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:13:54,195 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:13:56,198 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:13:58,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:00,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:00,320 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:00,320 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:00,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:01,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:01,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:01,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:01,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:02,255 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:03,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:05,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:05,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:05,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:05,402 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:06,258 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:08,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:09,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:10,093 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:10,093 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:10,094 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:10,099 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:10,262 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:10,409 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:14:12,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:15,411 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:16,389 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:16,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:16,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:16,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:16,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:17,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:18,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:20,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:20,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:20,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:20,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:20,700 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:21,273 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:22,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:25,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:27,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:27,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:27,437 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:27,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:28,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:30,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:31,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:32,056 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:32,056 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:32,056 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:32,081 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:32,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:34,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:36,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:36,517 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:36,517 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:36,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:36,522 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:37,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:38,305 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:40,414 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:14:42,415 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:45,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:45,028 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:45,029 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:45,033 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:45,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:46,313 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:46,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:14:46,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:14:46,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:14:47,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:50,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:50,246 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:50,246 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:50,251 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:50,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:52,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:53,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:14:55,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:14:55,781 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:14:55,781 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:14:55,786 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:14:56,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:14:58,324 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:14:58,787 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:01,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:01,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:01,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:01,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:01,718 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:01,719 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:01,724 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:02,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:04,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:04,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:06,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:06,225 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:06,225 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:06,230 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:06,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:08,333 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:10,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:10,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:15:11,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:11,223 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:11,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:11,229 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:11,335 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:12,337 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:16,230 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:16,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:16,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:16,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:16,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:16,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:16,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:17,341 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:18,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:21,874 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:21,875 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:21,875 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:21,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:21,880 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:22,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:24,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:26,881 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:27,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:27,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:27,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:27,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:28,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:30,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:31,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:31,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:31,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:32,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:32,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:32,966 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:32,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:32,972 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:33,359 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:34,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:38,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:38,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:38,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:38,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:38,714 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:39,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:40,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:40,418 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:15:43,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:43,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:43,930 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:43,931 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:43,937 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:44,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:46,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:46,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:15:46,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:15:46,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:15:48,253 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:48,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:48,254 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:48,259 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:48,374 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:49,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:50,377 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:53,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:53,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:53,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:53,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:15:54,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:15:54,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:15:56,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:15:59,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:15:59,577 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:15:59,577 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:15:59,582 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:00,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:00,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:01,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:01,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:01,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:02,388 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:04,595 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:04,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:04,595 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:04,600 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:05,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:05,636 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:06,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:08,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:08,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:08,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:08,895 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:09,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:10,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:10,417 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:16:11,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:13,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:13,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:13,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:13,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:14,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:16,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:16,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:16,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:16,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:16,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:18,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:18,101 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:18,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:18,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:18,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:20,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:22,018 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:22,019 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:22,019 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:22,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:22,024 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:22,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:24,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:27,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:30,273 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:30,274 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:30,274 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:30,279 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:30,415 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:31,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:31,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:31,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:32,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:32,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:35,347 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:35,348 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:35,348 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:35,353 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:35,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:36,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:38,388 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:40,419 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:16:40,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:40,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:40,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:40,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:41,426 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:42,428 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:43,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:45,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:45,533 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:45,533 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:45,538 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:46,389 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:16:46,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:16:46,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:16:46,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:46,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:49,601 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:50,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:50,410 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:50,410 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:50,415 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:50,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:16:52,450 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:16:55,452 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:16:58,375 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:16:58,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:16:58,376 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:16:58,381 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:16:58,453 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:00,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:01,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:01,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:01,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:01,390 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:03,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:03,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:03,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:03,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:04,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:06,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:07,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:08,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:08,454 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:08,454 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:08,459 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:08,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:10,422 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:17:10,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:12,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:12,963 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:12,964 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:12,964 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:12,969 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:13,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:14,471 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:16,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:16,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:16,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:17,258 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:17,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:17,259 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:17,265 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:17,474 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:18,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:18,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:21,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:21,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:21,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:21,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:22,479 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:23,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:24,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:26,267 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:26,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:26,268 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:26,273 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:26,483 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:28,486 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:29,274 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:30,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:30,661 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:30,662 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:30,667 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:31,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:31,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:31,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:32,491 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:34,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:35,916 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:35,916 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:35,917 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:35,921 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:36,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:38,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:39,960 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:40,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:17:40,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:40,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:40,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:40,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:41,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:42,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:45,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:46,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:46,129 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:46,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:46,134 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:46,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:17:46,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:17:46,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:17:46,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:48,508 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:51,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:51,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:51,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:51,245 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:51,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:51,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:52,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:17:56,007 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:17:56,008 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:17:56,008 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:17:56,013 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:17:56,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:17:57,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:17:58,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:00,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:00,841 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:00,841 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:00,846 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:01,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:01,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:01,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:01,526 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:02,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:02,641 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:06,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:06,296 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:06,296 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:06,302 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:06,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:08,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:08,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:10,429 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:18:13,243 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:13,244 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:13,244 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:13,249 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:13,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:14,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:14,540 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:16,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:16,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:16,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:17,547 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:17,548 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:17,548 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:17,553 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:18,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:18,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:19,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:22,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:22,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:22,452 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:22,457 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:22,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:24,550 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:25,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:27,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:27,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:27,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:27,397 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:27,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:28,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:31,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:31,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:32,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:32,504 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:32,504 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:32,509 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:32,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:34,609 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:36,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:38,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:38,530 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:38,530 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:38,535 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:38,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:40,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:18:40,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:42,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:43,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:43,458 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:43,458 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:43,462 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:43,616 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:44,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:46,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:18:46,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:18:46,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:18:47,657 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:52,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:52,652 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:52,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:52,658 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:52,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:53,624 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:18:54,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:18:57,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:18:58,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:18:58,326 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:18:58,326 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:18:58,331 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:18:58,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:00,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:01,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:01,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:01,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:03,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:04,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:04,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:04,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:04,173 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:04,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:06,638 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:08,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:08,619 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:08,619 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:08,624 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:08,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:09,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:10,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:19:10,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:12,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:12,998 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:12,998 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:13,003 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:13,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:14,646 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:15,004 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:16,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:16,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:16,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:18,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:18,337 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:18,337 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:18,343 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:18,650 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:20,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:20,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:25,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:26,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:26,728 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:26,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:26,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:27,722 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:28,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:30,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:31,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:33,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:33,854 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:33,855 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:33,860 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:34,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:35,860 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:36,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:39,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:39,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:39,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:39,030 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:39,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:40,432 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:19:40,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:41,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:43,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:43,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:43,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:43,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:43,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:44,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:46,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:19:46,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:19:46,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:19:46,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:48,743 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:48,744 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:48,744 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:48,749 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:49,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:50,748 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:51,788 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:54,906 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:54,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:54,907 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:54,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:55,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:19:56,754 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:19:56,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:19:59,631 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:19:59,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:19:59,633 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:19:59,638 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:19:59,756 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:00,758 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:01,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:01,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:01,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:02,637 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:03,824 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:03,825 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:03,825 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:03,830 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:04,761 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:06,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:07,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:10,437 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:20:13,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:16,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:16,152 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:16,152 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:16,157 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:16,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:16,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:16,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:16,771 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:18,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:18,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:21,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:21,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:21,819 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:21,824 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:22,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:23,825 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:24,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:26,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:26,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:26,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:26,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:26,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:28,819 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:29,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:31,163 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:31,164 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:31,164 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:31,170 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:31,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:31,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:31,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:32,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:34,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:37,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:37,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:37,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:37,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:37,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:38,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:40,188 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:40,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:20:41,538 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:41,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:41,539 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:41,544 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:41,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:42,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:45,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:45,566 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:45,567 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:45,567 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:45,572 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:45,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:46,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:20:46,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:20:46,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:20:46,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:50,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:52,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:52,508 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:52,508 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:52,513 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:52,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:54,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:20:56,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:20:57,571 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:20:57,572 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:20:57,572 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:20:57,577 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:20:57,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:20:58,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:01,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:01,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:01,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:01,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:02,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:02,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:02,404 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:02,409 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:02,853 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:04,856 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:07,410 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:08,793 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:08,794 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:08,794 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:08,799 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:08,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:10,438 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:21:10,861 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:12,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:14,451 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:14,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:14,452 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:14,457 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:14,864 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:16,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:16,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:16,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:16,868 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:17,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:19,013 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:19,014 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:19,014 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:19,019 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:19,870 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:20,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:23,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:24,957 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:24,958 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:24,959 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:24,964 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:25,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:26,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:28,964 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:29,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:29,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:29,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:29,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:29,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:30,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:31,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:31,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:31,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:34,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:38,115 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:38,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:38,116 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:38,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:38,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:40,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:40,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:21:40,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:42,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:42,739 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:42,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:42,745 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:42,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:44,909 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:45,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:46,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:21:46,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:21:46,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:21:47,563 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:47,564 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:47,564 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:47,569 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:47,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:48,913 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:51,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:52,447 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:52,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:52,448 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:52,453 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:52,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:54,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:21:56,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:21:56,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:21:56,896 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:21:56,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:21:56,901 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:21:56,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:21:58,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:01,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:01,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:01,393 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:01,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:01,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:01,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:01,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:01,940 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:02,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:02,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:06,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:06,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:06,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:06,225 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:06,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:08,226 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:08,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:10,443 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:22:11,616 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:11,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:11,617 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:11,623 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:11,950 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:12,953 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:13,624 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:16,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:16,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:16,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:18,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:18,088 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:18,088 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:18,093 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:18,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:19,094 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:20,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:23,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:23,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:23,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:23,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:23,963 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:24,357 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:24,966 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:29,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:29,783 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:29,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:29,784 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:29,789 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:29,968 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:30,971 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:31,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:31,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:31,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:34,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:34,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:34,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:34,059 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:34,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:35,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:36,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:40,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:40,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:40,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:40,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:40,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:40,448 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:22:40,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:42,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:45,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:46,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:46,046 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:46,047 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:46,052 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:46,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:22:46,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:22:46,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:22:46,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:49,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:50,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:51,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:51,491 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:51,492 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:51,497 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:52,049 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:53,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:22:56,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:22:56,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:22:56,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:22:56,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:22:56,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:22:57,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:22:59,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:01,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:01,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:01,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:01,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:04,749 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:04,750 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:04,750 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:04,755 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:05,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:06,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:07,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:10,447 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:23:12,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:13,241 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:13,241 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:13,242 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:13,248 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:14,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:15,070 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:16,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:16,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:16,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:17,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:19,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:19,977 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:19,977 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:19,983 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:20,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:21,077 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:22,983 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:25,391 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:25,392 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:25,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:25,398 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:26,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:27,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:28,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:30,058 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:30,058 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:30,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:30,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:30,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:31,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:31,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:31,393 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:33,088 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:33,663 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:34,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:34,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:34,976 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:34,981 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:35,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:37,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:38,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:40,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:40,397 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:40,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:40,403 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:40,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:23:41,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:43,100 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:44,453 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:45,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:45,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:45,470 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:45,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:46,101 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:46,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:23:46,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:23:46,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:23:47,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:49,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:50,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:50,077 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:50,077 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:50,082 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:50,106 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:53,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:23:55,083 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:23:57,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:23:57,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:23:57,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:23:57,655 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:23:58,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:23:59,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:00,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:01,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:01,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:01,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:02,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:02,147 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:02,147 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:02,152 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:03,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:05,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:06,152 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:06,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:06,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:06,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:06,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:07,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:09,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:10,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:24:10,865 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:10,866 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:10,866 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:10,871 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:11,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:11,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:13,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:16,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:16,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:16,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:17,684 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:19,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:19,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:19,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:19,096 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:19,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:21,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:23,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:24,927 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:24,928 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:24,928 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:24,933 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:25,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:27,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:28,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:31,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:31,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:31,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:33,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:33,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:33,098 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:33,103 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:33,217 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:34,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:35,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:37,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:37,552 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:37,552 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:37,557 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:38,223 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:39,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:39,557 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:40,453 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:24:43,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:43,717 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:43,717 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:43,722 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:44,230 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:44,722 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:45,232 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:46,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:24:46,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:24:46,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:24:48,716 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:48,720 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:48,720 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:48,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:49,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:49,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:51,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:52,981 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:52,982 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:52,982 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:52,988 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:53,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:54,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:24:55,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:24:57,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:24:57,736 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:24:57,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:24:57,741 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:24:58,251 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:24:59,253 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:00,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:01,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:01,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:01,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:02,421 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:02,421 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:02,422 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:02,427 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:03,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:05,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:06,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:07,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:07,421 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:07,421 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:07,427 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:08,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:09,261 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:10,455 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:25:11,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:12,137 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:12,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:12,138 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:12,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:12,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:13,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:16,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:16,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:16,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:16,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:17,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:17,999 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:18,000 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:18,005 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:18,269 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:19,272 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:22,005 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:22,247 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:22,248 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:22,248 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:22,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:22,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:25,277 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:27,254 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:30,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:30,704 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:30,704 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:30,709 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:31,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:31,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:31,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:31,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:32,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:33,297 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:36,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:36,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:36,112 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:36,117 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:36,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:37,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:38,118 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:40,457 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:25:40,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:40,536 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:40,536 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:40,541 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:41,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:43,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:43,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:45,109 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:45,110 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:45,110 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:45,115 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:45,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:46,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:25:46,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:25:46,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:25:47,314 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:48,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:49,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:49,616 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:49,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:49,621 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:50,315 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:51,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:53,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:25:53,890 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:25:53,890 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:25:53,891 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:25:53,896 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:25:54,320 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:25:55,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:25:58,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:01,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:01,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:01,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:01,919 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:01,919 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:01,920 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:01,925 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:02,326 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:03,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:03,925 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:06,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:06,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:06,238 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:06,243 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:06,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:09,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:09,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:10,459 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:26:14,461 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:14,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:14,593 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:14,593 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:14,598 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:15,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:16,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:16,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:16,394 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:17,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:19,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:20,010 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:20,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:20,011 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:20,016 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:20,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:21,344 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:24,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:24,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:24,527 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:24,533 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:25,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:25,534 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:26,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:28,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:28,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:28,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:28,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:29,377 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:30,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:30,773 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:31,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:31,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:31,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:33,282 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:33,283 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:33,283 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:33,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:33,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:34,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:36,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:38,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:38,147 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:38,147 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:38,152 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:38,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:38,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:40,461 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:26:41,463 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:42,951 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:42,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:42,952 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:42,957 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:43,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:44,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:46,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:26:46,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:26:46,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:26:46,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:48,351 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:48,352 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:48,352 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:48,357 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:48,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:50,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:52,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:52,818 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:52,819 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:52,819 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:52,825 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:53,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:26:54,405 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:57,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:26:57,443 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:26:57,444 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:26:57,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:26:57,455 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:26:58,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:26:58,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:01,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:01,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:01,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:02,169 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:02,170 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:02,170 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:02,176 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:02,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:02,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:03,176 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:06,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:06,402 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:06,403 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:06,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:06,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:08,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:08,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:10,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:27:13,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:13,081 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:13,081 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:13,086 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:13,422 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:14,087 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:14,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:16,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:16,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:16,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:19,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:21,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:21,621 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:21,621 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:21,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:22,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:22,432 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:25,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:27,150 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:27,152 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:27,152 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:27,157 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:27,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:28,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:31,157 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:31,393 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:31,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:31,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:33,043 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:33,044 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:33,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:33,050 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:33,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:34,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:37,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:37,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:37,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:37,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:37,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:38,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:38,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:40,466 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:27:42,467 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:44,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:44,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:44,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:44,051 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:44,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:44,455 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:46,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:27:46,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:27:46,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:27:47,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:50,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:50,754 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:50,754 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:50,759 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:51,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:27:52,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:27:52,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:57,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:27:58,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:27:58,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:27:58,805 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:27:58,810 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:27:59,476 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:00,480 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:01,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:01,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:01,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:03,680 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:04,760 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:04,761 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:04,761 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:04,767 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:05,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:06,505 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:08,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:09,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:09,579 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:09,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:09,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:10,471 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:28:10,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:10,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:14,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:16,343 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:16,344 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:16,345 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:16,350 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:16,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:16,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:16,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:16,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:16,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:19,604 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:21,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:21,032 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:21,032 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:21,037 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:21,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:22,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:25,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:25,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:25,870 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:25,870 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:25,875 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:26,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:26,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:30,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:31,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:31,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:31,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:31,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:31,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:31,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:31,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:32,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:32,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:36,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:39,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:39,565 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:39,565 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:39,570 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:39,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:40,470 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:28:40,683 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:42,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:44,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:44,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:44,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:44,445 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:44,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:46,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:28:46,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:28:46,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:28:46,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:47,704 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:48,968 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:48,969 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:48,969 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:48,974 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:49,690 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:50,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:52,975 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:53,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:53,714 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:53,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:53,719 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:54,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:28:54,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:28:58,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:28:58,589 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:28:58,590 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:28:58,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:28:58,595 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:28:58,699 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:00,702 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:01,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:01,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:01,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:03,705 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:07,096 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:07,097 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:07,097 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:07,102 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:07,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:08,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:09,102 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:10,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:29:11,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:11,493 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:11,494 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:11,499 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:11,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:12,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:14,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:16,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:16,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:16,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:17,470 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:17,471 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:17,471 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:17,476 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:17,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:18,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:20,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:23,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:23,147 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:23,147 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:23,152 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:23,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:24,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:26,153 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:28,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:28,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:28,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:28,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:28,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:28,730 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:31,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:31,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:31,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:31,396 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:32,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:32,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:32,719 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:32,725 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:32,732 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:34,736 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:36,725 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:39,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:39,033 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:39,033 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:39,039 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:39,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:40,475 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:29:40,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:42,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:43,323 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:43,324 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:43,325 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:43,330 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:43,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:44,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:46,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:29:46,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:29:46,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:29:47,632 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:47,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:47,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:47,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:47,804 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:48,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:48,751 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:52,804 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:29:56,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:29:56,325 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:29:56,326 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:29:56,330 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:29:56,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:29:56,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:29:58,331 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:01,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:01,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:01,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:02,374 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:02,374 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:02,375 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:02,380 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:02,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:02,763 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:03,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:07,307 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:07,308 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:07,309 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:07,314 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:07,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:08,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:09,314 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:10,476 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:30:14,478 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:15,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:15,813 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:15,813 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:15,818 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:16,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:16,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:16,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:16,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:16,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:19,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:20,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:20,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:20,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:20,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:20,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:20,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:24,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:24,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:24,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:24,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:24,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:25,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:26,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:29,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:29,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:29,469 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:29,474 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:29,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:30,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:31,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:31,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:31,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:31,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:36,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:36,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:36,330 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:36,335 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:36,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:36,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:37,335 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:40,482 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:30:42,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:42,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:42,922 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:42,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:42,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:43,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:44,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:46,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:30:46,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:30:46,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:30:47,312 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:47,313 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:47,313 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:47,318 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:47,806 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:48,319 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:48,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:51,570 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:51,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:51,571 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:51,576 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:51,810 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:52,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:53,576 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:30:55,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:30:55,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:30:55,834 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:30:55,839 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:30:56,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:30:56,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:30:58,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:00,341 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:00,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:00,342 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:00,347 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:00,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:00,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:01,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:01,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:01,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:04,645 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:07,091 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:07,092 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:07,092 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:07,097 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:07,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:08,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:10,098 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:10,480 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:31:10,885 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:10,886 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:10,887 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:10,893 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:11,829 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:12,833 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:15,893 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:16,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:16,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:16,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:16,655 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:16,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:16,656 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:16,662 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:16,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:18,840 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:21,662 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:25,187 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:25,188 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:25,188 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:25,193 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:25,843 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:26,846 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:27,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:29,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:29,475 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:29,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:29,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:29,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:30,850 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:31,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:31,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:31,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:32,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:33,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:33,472 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:33,472 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:33,477 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:33,852 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:34,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:38,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:40,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:40,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:40,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:40,373 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:40,487 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:31:40,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:40,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:44,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:45,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:45,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:45,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:45,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:45,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:46,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:31:46,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:31:46,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:31:46,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:49,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:52,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:52,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:52,941 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:52,946 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:53,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:54,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:31:54,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:31:57,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:31:57,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:31:57,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:31:57,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:31:57,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:31:58,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:00,442 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:01,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:01,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:01,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:01,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:01,859 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:01,859 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:01,865 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:01,911 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:02,914 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:05,865 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:06,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:06,252 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:06,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:06,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:06,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:06,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:10,486 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:32:11,487 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:13,955 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:13,956 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:13,956 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:13,961 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:13,988 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:14,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:16,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:16,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:16,397 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:16,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:18,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:18,748 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:18,748 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:18,754 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:18,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:20,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:21,754 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:23,711 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:23,712 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:23,712 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:23,718 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:24,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:25,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:27,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:30,817 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:30,818 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:30,818 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:30,823 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:31,005 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:31,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:31,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:31,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:33,008 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:33,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:36,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:36,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:36,281 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:36,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:37,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:37,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:39,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:40,491 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:32:41,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:41,265 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:41,266 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:41,271 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:42,018 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:43,020 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:45,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:46,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:46,054 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:46,054 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:46,059 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:46,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:32:46,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:32:46,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:32:47,024 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:47,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:50,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:51,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:51,277 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:51,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:51,283 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:52,027 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:53,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:32:56,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:32:57,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:32:57,369 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:32:57,369 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:32:57,374 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:32:58,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:32:59,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:01,375 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:01,395 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:01,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:01,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:01,877 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:01,877 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:01,883 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:02,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:03,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:06,206 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:06,206 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:06,207 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:06,212 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:07,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:07,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:07,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:10,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:10,233 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:10,233 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:10,238 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:10,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:33:11,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:11,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:12,492 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:16,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:16,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:16,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:17,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:17,456 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:17,457 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:17,461 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:18,053 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:18,462 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:19,055 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:22,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:22,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:22,159 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:22,164 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:23,059 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:23,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:24,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:26,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:26,852 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:26,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:26,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:27,062 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:29,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:29,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:31,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:31,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:32,987 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:32,988 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:32,988 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:32,993 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:33,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:34,993 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:35,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:37,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:37,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:37,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:37,206 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:38,073 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:39,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:40,207 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:40,492 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:33:42,321 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:42,326 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:42,326 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:42,331 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:43,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:43,084 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:45,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:46,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:33:46,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:33:46,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:33:48,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:48,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:48,178 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:48,184 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:49,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:49,089 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:51,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:52,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:52,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:52,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:52,400 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:53,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:33:53,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:56,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:33:57,630 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:33:57,631 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:33:57,631 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:33:57,636 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:33:58,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:33:59,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:01,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:34:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:34:01,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:01,674 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:03,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:03,475 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:03,475 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:03,480 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:04,102 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:05,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:07,481 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:08,174 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:08,175 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:08,175 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:08,180 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:09,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:09,109 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:10,494 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:34:12,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:13,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:13,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:13,159 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:13,164 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:14,112 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:15,114 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:16,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:34:16,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:34:16,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:17,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:17,694 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:17,694 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:17,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:17,699 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:18,117 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:19,120 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:21,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:21,775 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:21,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:21,780 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:22,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:22,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:23,124 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:26,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:26,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:26,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:26,774 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:27,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:27,130 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:28,774 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:31,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:34:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:34:31,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:31,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:31,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:31,654 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:31,659 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:32,133 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:33,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:34,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:35,614 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:35,615 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:35,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:35,620 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:36,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:37,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:40,497 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:34:40,498 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:41,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:41,052 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:41,053 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:41,058 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:41,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:43,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:46,059 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:46,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:34:46,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:34:46,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:34:46,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:46,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:46,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:46,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:47,149 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:49,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:51,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:53,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:53,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:53,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:53,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:54,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:55,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:34:57,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:34:57,795 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:34:57,795 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:34:57,796 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:34:57,800 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:34:58,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:34:59,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:01,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:35:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:01,398 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:02,541 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:02,542 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:02,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:02,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:03,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:03,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:03,548 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:08,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:09,097 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:09,098 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:09,099 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:09,104 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:09,183 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:10,499 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:35:11,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:14,233 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:14,234 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:14,234 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:14,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:14,239 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:15,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:15,196 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:16,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:35:16,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:16,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:19,706 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:19,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:19,715 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:19,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:19,720 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:20,205 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:21,207 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:24,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:24,708 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:24,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:24,709 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:24,714 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:25,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:25,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:29,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:29,250 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:29,250 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:29,256 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:30,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:30,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:31,243 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:31,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:35:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:31,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:33,430 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:33,431 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:33,431 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:33,437 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:34,244 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:35,249 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:35,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:39,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:39,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:39,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:39,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:40,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:40,501 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:35:40,502 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:41,254 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:43,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:43,849 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:43,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:43,855 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:44,257 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:45,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:45,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:46,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:35:46,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:35:46,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:35:51,642 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:52,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:52,731 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:52,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:52,737 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:53,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:53,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:35:56,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:35:58,910 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:35:58,911 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:35:58,911 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:35:58,915 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:35:59,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:35:59,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:01,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:36:01,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:01,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:02,618 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:03,537 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:03,538 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:03,538 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:03,542 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:04,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:05,300 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:07,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:07,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:07,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:07,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:07,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:08,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:09,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:10,506 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:36:13,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:15,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:15,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:15,940 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:15,945 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:16,309 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:16,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:36:16,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:16,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:17,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:18,627 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:21,843 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:21,844 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:21,844 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:21,849 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:22,329 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:23,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:23,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:26,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:26,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:26,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:26,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:27,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:27,338 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:29,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:30,727 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:30,728 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:30,728 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:30,733 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:31,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:31,343 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:31,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:36:31,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:31,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:34,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:35,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:35,747 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:35,747 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:35,752 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:36,347 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:37,349 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:39,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:40,505 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:36:40,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:40,976 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:40,977 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:40,982 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:41,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:43,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:44,982 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:45,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:45,169 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:45,169 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:45,174 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:45,357 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:46,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:36:46,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:36:46,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:36:47,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:50,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:50,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:50,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:50,833 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:50,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:51,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:51,364 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:55,086 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:55,087 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:55,087 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:55,092 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:55,372 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:36:56,092 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:36:57,375 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:58,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:36:58,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:36:58,938 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:36:58,943 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:36:59,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:36:59,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:01,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:37:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:01,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:01,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:05,154 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:05,155 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:05,155 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:05,160 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:05,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:07,161 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:07,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:09,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:09,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:09,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:09,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:10,387 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:10,510 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:37:11,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:12,511 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:15,469 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:15,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:15,470 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:15,475 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:16,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:16,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:37:16,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:16,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:17,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:17,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:21,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:21,165 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:21,166 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:21,170 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:21,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:23,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:23,403 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:26,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:26,598 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:26,598 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:26,603 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:27,408 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:28,603 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:29,413 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:31,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:31,202 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:31,202 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:31,207 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:31,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:37:31,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:31,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:31,414 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:33,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:33,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:35,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:35,891 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:35,891 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:35,897 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:36,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:37,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:38,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:40,509 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:37:41,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:41,366 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:41,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:41,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:41,431 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:43,437 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:44,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:46,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:37:46,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:37:46,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:37:48,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:48,697 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:48,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:48,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:49,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:49,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:51,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:54,703 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:37:55,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:55,215 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:55,215 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:55,220 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:37:55,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:37:57,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:37:59,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:37:59,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:37:59,641 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:37:59,646 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:00,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:00,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:01,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:38:01,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:38:01,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:38:01,459 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:05,683 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:06,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:06,614 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:06,615 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:06,621 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:07,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:09,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:10,515 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:38:11,516 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:14,564 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:14,565 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:14,565 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:14,570 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:15,471 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:15,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:16,397 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:38:16,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:38:16,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:38:16,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:19,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:19,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:19,285 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:19,290 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:19,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:21,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:22,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:24,113 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:24,113 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:24,114 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:24,119 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:24,480 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:25,483 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:28,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:28,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:28,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:28,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:28,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:28,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:29,488 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:38:31,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:38:31,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:38:33,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:33,127 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:33,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:33,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:33,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:33,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:35,500 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:38,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:39,814 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:39,815 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:39,815 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:39,820 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:40,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:40,513 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:38:41,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:43,515 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:45,370 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:45,371 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:45,371 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:45,376 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:45,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:46,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:38:46,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:38:46,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:38:47,512 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:48,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:50,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:50,211 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:50,211 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:50,217 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:50,514 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:51,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:54,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:38:55,466 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:38:55,467 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:38:55,467 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:38:55,474 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:38:55,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:38:57,524 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:38:59,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:01,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:01,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:01,036 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:01,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:39:01,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:39:01,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:39:01,527 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:03,534 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:04,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:06,132 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:06,133 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:06,133 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:06,139 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:06,537 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:07,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:10,140 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:10,515 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:39:10,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:10,796 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:10,797 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:10,802 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:11,547 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:13,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:15,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:15,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:15,781 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:15,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:15,788 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:16,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:39:16,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:39:16,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:39:16,554 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:17,560 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:20,583 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:20,584 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:20,584 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:20,591 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:21,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:21,570 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:21,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:25,066 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:25,066 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:25,067 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:25,073 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:25,573 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:27,074 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:27,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:39:31,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:39:31,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:39:31,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:31,605 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:31,605 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:31,613 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:32,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:32,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:33,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:36,104 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:36,105 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:36,106 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:36,116 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:36,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:37,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:38,116 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:40,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:39:40,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:40,840 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:40,841 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:40,849 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:41,605 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:43,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:43,849 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:45,989 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:45,990 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:45,990 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:45,996 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:46,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:39:46,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:39:46,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:39:46,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:47,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:49,665 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:50,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:50,784 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:50,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:50,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:51,622 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:53,628 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:39:54,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:39:56,171 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:39:56,172 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:39:56,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:39:56,179 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:39:56,630 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:39:57,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:00,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:00,340 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:00,341 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:00,341 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:00,349 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:00,635 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:40:01,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:40:01,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:40:01,640 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:05,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:05,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:05,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:05,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:05,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:06,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:07,647 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:10,158 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:10,159 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:10,159 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:10,166 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:10,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:40:10,650 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:11,521 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:11,654 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:15,383 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:15,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:15,384 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:15,390 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:15,657 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:16,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:40:16,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:40:16,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:40:16,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:17,664 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:20,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:20,590 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:20,590 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:20,596 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:20,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:21,671 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:22,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:27,378 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:27,379 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:27,379 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:27,387 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:27,675 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:28,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:29,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:40:31,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:40:31,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:40:31,834 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:31,835 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:31,836 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:31,843 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:32,682 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:33,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:33,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:36,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:36,687 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:36,687 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:36,694 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:36,694 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:37,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:39,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:40,522 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:40:41,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:41,137 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:41,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:41,143 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:41,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:43,709 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:45,143 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:46,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:40:46,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:40:46,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:40:48,477 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:48,478 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:48,478 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:48,485 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:48,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:49,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:50,485 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:40:53,125 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:40:53,126 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:40:53,126 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:40:53,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:40:53,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:40:55,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:40:56,133 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:01,134 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:01,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:41:01,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:41:01,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:41:01,507 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:01,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:01,618 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:01,626 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:01,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:03,734 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:05,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:05,973 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:05,973 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:05,980 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:06,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:06,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:07,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:10,524 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:41:11,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:11,072 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:11,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:11,079 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:11,746 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:12,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:13,749 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:16,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:41:16,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:41:16,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:41:16,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:16,663 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:16,663 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:16,670 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:16,752 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:17,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:17,757 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:22,231 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:22,231 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:22,231 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:22,238 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:22,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:23,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:23,765 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:28,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:30,643 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:30,643 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:30,644 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:30,650 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:30,770 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:31,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:41:31,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:41:31,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:41:31,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:33,613 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:35,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:35,053 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:35,053 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:35,060 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:35,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:37,785 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:39,060 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:39,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:39,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:39,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:39,791 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:40,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:41:40,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:41,790 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:44,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:44,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:44,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:44,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:44,410 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:44,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:45,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:46,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:41:46,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:41:46,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:41:48,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:48,653 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:48,653 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:48,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:48,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:49,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:49,804 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:54,661 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:41:56,669 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:41:56,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:41:56,670 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:41:56,678 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:41:56,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:41:57,813 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:41:59,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:01,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:42:01,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:42:01,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:42:02,437 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:02,438 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:02,438 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:02,444 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:02,817 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:03,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:05,444 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:08,290 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:08,291 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:08,291 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:08,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:08,824 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:09,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:10,528 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:42:10,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:15,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:16,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:42:16,401 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:42:16,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:42:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:16,771 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:16,772 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:16,785 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:16,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:17,839 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:20,785 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:21,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:21,475 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:21,476 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:21,483 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:21,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:23,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:25,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:25,714 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:25,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:25,721 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:25,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:26,721 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:27,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:30,136 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:30,136 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:30,137 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:30,142 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:30,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:31,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:42:31,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:42:31,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:42:31,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:32,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:37,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:38,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:38,638 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:38,639 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:38,644 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:38,866 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:39,869 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:40,533 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:42:42,535 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:43,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:43,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:43,647 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:43,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:43,873 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:45,877 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:46,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:42:46,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:42:46,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:42:48,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:48,256 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:48,257 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:48,257 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:48,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:48,879 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:49,882 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:53,300 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:54,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:54,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:54,647 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:54,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:54,885 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:42:55,889 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:42:58,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:42:59,318 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:42:59,319 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:42:59,319 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:42:59,325 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:42:59,892 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:01,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:43:01,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:43:01,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:43:01,895 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:04,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:04,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:04,111 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:04,111 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:04,117 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:04,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:05,901 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:07,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:07,974 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:07,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:07,979 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:08,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:09,906 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:09,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:10,532 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:43:12,433 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:12,434 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:12,434 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:12,440 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:12,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:13,912 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:15,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:16,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:43:16,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:43:16,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:43:17,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:17,599 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:17,599 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:17,604 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:17,916 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:19,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:20,605 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:22,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:22,296 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:22,296 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:22,301 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:22,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:23,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:26,302 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:27,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:27,489 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:27,489 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:27,494 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:27,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:29,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:31,495 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:31,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:43:31,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:43:31,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:43:32,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:32,699 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:32,699 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:32,704 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:32,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:33,935 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:36,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:37,717 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:37,718 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:37,718 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:37,723 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:37,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:39,942 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:40,534 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:43:42,536 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:42,939 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:42,940 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:42,940 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:42,945 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:42,945 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:43,948 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:46,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:43:46,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:43:46,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:43:47,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:47,957 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:47,957 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:47,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:47,962 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:48,952 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:49,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:52,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:52,215 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:52,216 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:52,221 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:52,956 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:43:53,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:53,958 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:43:58,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:43:59,191 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:43:59,192 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:43:59,192 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:43:59,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:43:59,962 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:01,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:44:01,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:44:01,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:44:01,965 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:03,868 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:03,868 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:03,869 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:03,869 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:03,874 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:03,966 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:05,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:08,912 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:10,536 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:44:12,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:12,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:12,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:12,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:12,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:13,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:14,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:16,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:44:16,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:44:16,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:44:17,363 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:17,364 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:17,364 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:17,369 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:17,979 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:19,408 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:19,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:21,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:21,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:21,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:21,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:21,983 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:23,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:24,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:27,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:27,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:27,851 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:27,856 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:27,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:29,857 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:29,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:31,731 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:44:31,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:44:31,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:44:32,454 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:32,455 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:32,455 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:32,460 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:32,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:33,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:35,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:38,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:38,648 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:38,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:38,653 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:39,001 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:40,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:40,538 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:44:40,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:43,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:43,272 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:43,272 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:43,277 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:44,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:46,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:46,278 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:46,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:44:46,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:44:46,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:44:48,284 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:48,284 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:48,284 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:48,289 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:49,012 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:50,014 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:51,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:52,792 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:52,793 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:52,793 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:52,798 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:53,016 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:44:54,019 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:44:56,798 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:44:58,129 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:44:58,130 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:44:58,130 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:44:58,135 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:44:59,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:00,025 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:01,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:45:01,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:45:01,732 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:45:02,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:02,898 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:02,899 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:02,899 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:02,904 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:03,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:04,029 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:07,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:10,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:45:10,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:10,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:10,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:10,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:11,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:12,035 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:13,838 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:15,553 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:15,554 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:15,554 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:15,559 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:16,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:16,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:45:16,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:45:16,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:45:18,042 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:19,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:20,587 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:20,587 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:20,588 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:20,592 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:21,127 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:22,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:24,628 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:25,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:25,196 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:25,196 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:25,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:26,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:28,135 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:30,202 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:30,678 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:30,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:30,679 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:30,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:31,137 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:31,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:45:31,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:45:31,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:45:32,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:36,011 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:36,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:36,529 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:36,530 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:36,535 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:37,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:38,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:40,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:45:40,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:40,841 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:40,842 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:40,847 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:41,148 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:41,847 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:42,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:46,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:45:46,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:45:46,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:45:47,021 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:49,371 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:49,372 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:49,372 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:49,377 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:50,155 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:52,159 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:45:52,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:45:57,197 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:45:57,198 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:45:57,198 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:45:57,203 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:45:58,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:45:58,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:00,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:01,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:46:01,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:46:01,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:46:02,517 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:02,518 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:02,518 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:02,523 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:03,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:03,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:04,170 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:07,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:07,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:07,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:07,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:07,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:08,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:09,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:10,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:46:11,331 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:11,332 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:11,332 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:11,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:12,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:14,180 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:14,338 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:15,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:15,678 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:15,678 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:15,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:16,181 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:16,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:46:16,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:46:16,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:46:18,186 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:20,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:20,026 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:20,026 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:20,026 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:20,031 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:20,187 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:22,190 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:24,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:24,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:24,289 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:24,294 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:25,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:25,295 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:26,194 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:29,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:29,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:29,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:29,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:30,197 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:30,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:31,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:46:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:46:31,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:46:32,200 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:35,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:35,221 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:35,221 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:35,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:36,203 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:36,264 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:38,206 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:40,547 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:46:40,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:40,996 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:40,996 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:41,001 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:41,208 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:42,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:42,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:46,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:46:46,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:46:46,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:46:47,998 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:48,364 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:48,365 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:48,365 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:48,370 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:49,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:50,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:52,995 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:46:52,996 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:46:52,996 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:46:53,001 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:46:53,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:46:53,219 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:46:54,222 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:46:58,002 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:00,089 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:00,090 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:00,090 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:00,095 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:00,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:01,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:47:01,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:47:01,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:47:02,229 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:03,956 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:04,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:04,477 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:04,477 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:04,481 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:05,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:06,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:09,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:10,552 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:47:12,299 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:12,300 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:12,300 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:12,305 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:13,238 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:14,240 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:15,305 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:16,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:47:16,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:47:16,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:47:20,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:20,531 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:20,531 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:20,531 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:20,536 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:21,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:22,248 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:24,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:24,729 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:24,729 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:24,734 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:25,265 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:25,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:26,268 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:30,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:47:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:47:31,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:47:32,233 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:32,233 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:32,233 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:32,239 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:32,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:34,274 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:36,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:37,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:37,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:37,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:37,913 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:38,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:40,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:40,551 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:47:41,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:43,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:43,701 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:43,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:43,706 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:44,284 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:46,286 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:46,707 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:46,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:47:46,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:47:46,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:47:50,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:50,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:50,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:50,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:51,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:51,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:52,291 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:47:56,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:47:56,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:47:56,924 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:47:56,924 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:47:56,929 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:47:57,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:47:58,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:01,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:48:01,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:48:01,733 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:48:01,856 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:02,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:02,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:02,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:02,067 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:02,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:04,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:06,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:06,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:06,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:06,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:07,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:07,359 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:08,306 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:10,553 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:48:10,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:10,714 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:10,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:10,719 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:11,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:12,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:12,719 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:15,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:15,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:15,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:15,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:16,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:16,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:48:16,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:48:16,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:48:18,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:18,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:21,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:21,206 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:21,206 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:21,211 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:21,318 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:22,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:23,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:25,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:25,113 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:25,113 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:25,118 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:25,369 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:26,372 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:29,119 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:29,195 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:29,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:29,196 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:29,201 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:29,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:30,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:48:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:48:31,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:48:34,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:34,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:34,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:34,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:34,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:35,379 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:36,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:39,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:40,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:40,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:40,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:40,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:40,385 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:40,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:48:42,389 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:45,248 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:45,249 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:45,249 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:45,249 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:45,254 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:45,391 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:46,393 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:46,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:48:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:48:46,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:48:51,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:51,611 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:51,611 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:51,611 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:51,616 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:52,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:54,400 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:48:56,598 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:48:56,599 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:48:56,599 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:48:56,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:48:56,604 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:48:57,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:48:58,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:01,027 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:01,027 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:01,028 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:01,033 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:01,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:01,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:49:01,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:01,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:49:01,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:49:02,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:07,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:09,599 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:09,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:09,600 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:09,605 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:10,417 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:10,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:49:12,420 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:12,559 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:14,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:14,229 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:14,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:14,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:14,421 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:16,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:16,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:49:16,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:49:16,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:49:18,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:20,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:20,289 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:20,290 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:20,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:20,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:22,430 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:23,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:25,196 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:25,197 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:25,197 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:25,202 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:25,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:26,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:29,203 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:30,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:30,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:30,968 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:30,973 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:31,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:49:31,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:49:31,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:49:32,466 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:34,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:35,390 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:35,390 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:35,391 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:35,396 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:35,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:38,473 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:40,432 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:40,563 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:49:41,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:41,978 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:41,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:41,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:42,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:44,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:45,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:46,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:49:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:49:46,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:49:47,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:47,881 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:47,881 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:47,886 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:48,481 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:50,484 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:51,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:54,589 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:54,590 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:54,590 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:54,595 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:55,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:49:56,497 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:49:57,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:49:58,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:49:58,449 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:49:58,449 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:49:58,454 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:49:58,498 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:00,501 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:01,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:50:01,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:50:01,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:50:02,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:02,701 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:02,701 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:02,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:02,707 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:03,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:04,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:07,744 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:07,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:07,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:07,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:07,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:08,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:10,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:10,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:50:12,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:12,071 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:12,072 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:12,076 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:12,510 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:13,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:14,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:16,335 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:16,336 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:16,336 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:16,341 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:16,515 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:16,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:50:16,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:50:16,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:50:18,519 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:19,052 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:20,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:20,456 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:20,456 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:20,461 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:20,520 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:22,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:24,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:25,970 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:25,971 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:25,971 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:25,976 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:26,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:28,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:29,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:30,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:30,399 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:30,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:30,404 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:30,543 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:31,733 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:50:31,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:50:31,734 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:50:32,546 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:35,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:35,797 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:35,798 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:35,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:35,803 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:36,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:38,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:40,566 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:50:40,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:41,765 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:41,766 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:41,766 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:41,771 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:42,555 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:44,559 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:45,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:50:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:50:46,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:50:47,412 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:47,413 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:47,413 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:47,418 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:47,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:50,565 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:51,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:53,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:53,127 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:53,127 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:53,132 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:53,566 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:50:54,569 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:50:57,133 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:50:58,847 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:50:58,848 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:50:58,848 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:50:58,853 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:50:59,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:00,621 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:01,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:51:01,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:51:01,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:51:03,068 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:03,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:03,880 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:03,880 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:03,885 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:04,623 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:06,626 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:08,924 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:10,566 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:51:10,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:10,856 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:10,856 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:10,861 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:11,629 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:12,631 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:14,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:15,316 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:15,316 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:15,317 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:15,322 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:15,633 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:16,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:16,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:51:16,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:51:16,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:51:19,934 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:19,935 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:19,935 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:19,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:19,940 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:20,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:22,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:24,773 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:24,774 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:24,775 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:24,779 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:25,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:25,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:26,646 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:30,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:30,844 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:30,845 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:30,845 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:30,850 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:31,719 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:31,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:51:31,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:51:31,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:51:32,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:35,331 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:35,332 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:35,332 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:35,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:35,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:36,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:36,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:39,787 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:39,788 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:39,788 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:39,793 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:40,571 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:51:40,729 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:41,572 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:42,733 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:44,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:44,952 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:44,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:44,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:45,735 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:51:46,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:51:46,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:46,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:51:46,737 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:50,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:50,355 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:50,355 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:50,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:50,739 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:52,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:52,742 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:54,593 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:54,594 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:54,594 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:54,599 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:54,744 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:51:56,747 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:51:57,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:51:59,008 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:51:59,009 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:51:59,009 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:51:59,014 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:51:59,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:00,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:01,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:52:01,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:52:01,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:52:03,073 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:03,238 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:03,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:03,238 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:03,243 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:03,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:04,781 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:08,049 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:08,050 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:08,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:08,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:08,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:09,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:10,570 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:52:10,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:12,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:12,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:12,514 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:12,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:12,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:14,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:14,791 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:16,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:52:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:52:16,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:52:17,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:17,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:17,329 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:17,333 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:17,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:18,796 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:20,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:23,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:23,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:23,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:23,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:23,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:24,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:26,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:28,554 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:28,555 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:28,555 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:28,560 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:28,803 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:30,807 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:31,561 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:31,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:52:31,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:52:31,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:52:33,897 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:33,898 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:33,898 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:33,903 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:34,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:36,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:36,940 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:40,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:40,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:40,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:40,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:40,577 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:52:40,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:42,578 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:42,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:45,659 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:45,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:45,660 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:45,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:45,822 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:46,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:52:46,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:52:46,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:52:48,063 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:48,825 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:49,904 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:49,905 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:49,905 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:49,909 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:50,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:52,830 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:53,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:54,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:54,180 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:54,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:54,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:54,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:52:56,834 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:52:59,186 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:52:59,442 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:52:59,442 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:52:59,443 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:52:59,448 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:52:59,838 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:00,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:01,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:53:01,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:53:01,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:53:05,006 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:06,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:06,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:06,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:06,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:06,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:08,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:10,080 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:10,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:53:14,280 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:14,281 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:14,281 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:14,286 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:14,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:15,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:53:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:53:16,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:53:16,854 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:18,811 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:18,811 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:18,811 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:18,817 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:18,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:20,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:20,858 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:23,612 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:23,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:23,613 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:23,618 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:23,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:24,862 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:26,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:31,619 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:31,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:53:31,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:53:31,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:53:32,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:32,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:32,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:32,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:32,874 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:34,875 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:36,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:36,574 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:36,574 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:36,579 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:36,878 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:37,580 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:38,881 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:40,577 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:53:41,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:41,049 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:41,049 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:41,054 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:41,883 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:42,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:43,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:45,317 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:45,318 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:45,318 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:45,323 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:45,888 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:46,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:53:46,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:53:46,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:53:46,890 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:48,061 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:51,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:51,953 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:51,953 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:51,958 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:52,894 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:54,001 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:53:54,897 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:56,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:53:56,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:53:56,551 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:53:56,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:53:56,898 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:53:58,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:53:59,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:01,077 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:01,078 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:01,078 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:01,083 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:01,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:54:01,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:54:01,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:54:01,907 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:02,910 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:05,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:06,208 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:06,209 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:06,209 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:06,214 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:06,915 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:08,918 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:10,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:10,579 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:54:10,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:10,842 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:10,842 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:10,847 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:10,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:12,922 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:15,112 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:15,113 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:15,113 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:15,118 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:15,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:16,118 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:54:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:54:16,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:54:16,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:19,513 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:19,514 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:19,514 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:19,519 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:19,930 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:20,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:21,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:26,556 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:27,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:27,224 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:27,225 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:27,230 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:27,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:28,939 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:31,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:31,715 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:31,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:31,715 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:31,720 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:31,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:54:31,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:54:31,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:54:31,977 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:34,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:36,708 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:36,709 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:36,709 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:36,714 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:36,984 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:37,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:38,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:40,581 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:54:41,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:41,539 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:41,540 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:41,545 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:41,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:42,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:43,545 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:45,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:45,946 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:45,946 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:45,952 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:45,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:46,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:54:46,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:54:46,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:54:48,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:48,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:52,441 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:52,442 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:52,442 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:52,447 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:53,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:54,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:54:55,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:54:57,159 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:54:57,160 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:54:57,160 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:54:57,165 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:54:58,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:54:59,007 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:00,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:01,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:55:01,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:55:01,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:55:02,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:02,059 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:02,059 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:02,064 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:03,010 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:05,013 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:06,064 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:07,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:07,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:07,948 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:07,953 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:08,018 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:10,586 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:55:11,021 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:11,588 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:12,023 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:12,024 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:12,024 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:12,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:13,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:15,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:16,735 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:55:16,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:16,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:55:16,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:55:17,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:17,769 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:17,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:17,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:18,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:21,032 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:21,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:26,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:26,831 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:26,832 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:26,832 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:26,837 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:27,036 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:29,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:31,422 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:31,423 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:31,423 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:31,428 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:31,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:55:31,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:55:31,736 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:55:32,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:32,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:33,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:36,127 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:36,128 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:36,128 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:36,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:37,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:37,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:39,069 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:40,585 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:55:41,884 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:41,885 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:41,885 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:41,890 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:42,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:42,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:45,076 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:46,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:55:46,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:55:46,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:55:47,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:47,480 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:47,480 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:47,485 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:48,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:48,524 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:49,080 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:52,108 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:52,109 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:52,109 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:52,114 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:53,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:54,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:55:55,086 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:56,734 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:55:56,735 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:55:56,735 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:55:56,740 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:55:57,087 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:55:59,090 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:55:59,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:01,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:56:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:56:01,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:56:02,880 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:02,881 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:02,881 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:02,886 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:03,093 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:04,887 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:05,096 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:07,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:07,849 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:07,849 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:07,854 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:08,142 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:10,590 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:56:10,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:11,146 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:12,403 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:12,404 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:12,404 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:12,408 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:13,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:15,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:16,409 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:16,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:56:16,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:56:16,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:56:16,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:17,003 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:17,003 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:17,008 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:17,152 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:19,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:22,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:22,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:22,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:22,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:22,870 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:23,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:25,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:27,542 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:27,543 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:27,543 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:27,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:27,548 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:28,163 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:29,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:31,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:56:31,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:56:31,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:56:32,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:32,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:32,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:32,830 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:32,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:33,168 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:35,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:37,687 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:37,688 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:37,688 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:37,693 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:38,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:38,178 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:38,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:40,590 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:56:44,591 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:46,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:56:46,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:56:46,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:56:49,252 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:49,253 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:49,253 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:49,258 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:50,225 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:50,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:50,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:54,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:54,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:54,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:54,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:55,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:56:56,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:56:56,250 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:56:58,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:56:58,167 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:56:58,168 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:56:58,173 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:56:58,232 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:00,235 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:01,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:57:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:57:01,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:57:02,789 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:02,790 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:02,790 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:02,795 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:03,237 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:04,239 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:06,832 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:09,173 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:09,174 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:09,174 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:09,179 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:09,243 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:10,245 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:10,595 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:57:12,596 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:16,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:57:16,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:57:16,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:57:17,648 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:17,649 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:17,649 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:17,649 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:17,654 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:18,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:18,252 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:22,172 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:22,173 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:22,173 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:22,178 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:22,256 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:23,179 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:24,259 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:27,220 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:27,221 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:27,221 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:27,226 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:27,260 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:28,227 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:28,263 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:31,654 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:31,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:31,655 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:31,660 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:31,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:57:31,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:57:31,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:57:32,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:32,267 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:34,025 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:36,555 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:36,556 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:36,556 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:36,561 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:37,271 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:38,282 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:39,562 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:40,594 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:57:41,214 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:41,215 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:41,215 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:41,221 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:41,283 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:42,286 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:45,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:46,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:46,377 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:46,377 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:46,382 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:46,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:57:46,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:57:46,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:57:47,290 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:48,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:50,941 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:57:51,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:51,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:51,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:51,042 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:51,294 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:52,296 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:55,664 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:57:55,665 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:57:55,665 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:57:55,670 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:57:56,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:57:56,301 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:57:56,671 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:01,135 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:01,135 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:01,136 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:01,141 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:01,304 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:01,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:58:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:58:01,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:58:02,307 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:07,020 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:07,335 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:07,336 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:07,336 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:07,341 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:08,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:08,312 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:10,596 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:58:11,848 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:11,849 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:11,850 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:11,854 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:12,316 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:12,317 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:12,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:16,736 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:58:16,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:58:16,737 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:58:17,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:17,876 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:17,876 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:17,876 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:17,882 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:18,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:18,322 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:22,923 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:24,854 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:24,855 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:24,855 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:24,860 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:25,327 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:26,330 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:28,861 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:29,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:29,304 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:29,305 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:29,310 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:29,331 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:30,334 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:31,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:58:31,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:58:31,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:58:33,974 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:37,557 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:37,557 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:37,557 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:37,562 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:38,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:38,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:39,600 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:40,598 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:58:42,346 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:42,346 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:42,346 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:42,352 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:43,345 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:44,348 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:45,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:46,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:58:46,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:58:46,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:58:46,961 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:46,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:46,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:46,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:47,350 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:48,353 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:50,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:54,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:54,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:54,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:54,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:58:54,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:58:56,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:58:56,363 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:58:59,493 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:58:59,494 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:58:59,494 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:58:59,499 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:00,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:00,367 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:01,499 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:59:01,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:59:01,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:59:04,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:04,742 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:04,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:04,747 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:05,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:06,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:06,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:10,600 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:59:11,114 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:11,115 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:11,115 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:11,120 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:11,376 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:12,121 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:12,378 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:16,117 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:16,118 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:16,119 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:16,124 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:16,381 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:16,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:59:16,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:59:16,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:59:17,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:18,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:20,350 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:20,350 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:20,350 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:20,356 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:20,386 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:22,390 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:23,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:28,392 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:28,651 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:28,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:28,652 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:28,657 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:29,394 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:30,397 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:31,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:59:31,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:59:31,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:59:34,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:34,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:34,303 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:34,304 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:34,309 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:34,399 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:36,402 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:38,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:38,216 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:38,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:38,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:38,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:39,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:40,443 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:40,602 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 18:59:42,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:42,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:42,648 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:42,652 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:43,446 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:44,449 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:44,653 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:46,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 18:59:46,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 18:59:46,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 18:59:48,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:48,185 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:48,185 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:48,190 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:48,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:50,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:50,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:53,399 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:53,400 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:53,400 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:53,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:53,456 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 18:59:54,458 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 18:59:55,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 18:59:58,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 18:59:58,077 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 18:59:58,077 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 18:59:58,082 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 18:59:58,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:00,463 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:01,083 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:00:01,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:00:01,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:00:02,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:02,773 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:02,773 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:02,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:03,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:04,468 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:06,778 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:07,085 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:07,086 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:07,086 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:07,091 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:07,470 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:08,472 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:10,605 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:00:12,606 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:13,579 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:13,580 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:13,580 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:13,585 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:14,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:14,478 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:16,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:00:16,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:00:16,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:00:18,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:20,281 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:20,282 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:20,282 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:20,287 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:20,482 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:22,485 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:23,324 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:25,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:25,686 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:25,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:25,692 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:26,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:26,489 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:28,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:30,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:30,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:30,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:30,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:31,492 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:31,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:00:31,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:00:31,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:00:32,495 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:34,032 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:35,398 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:35,399 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:35,399 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:35,404 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:35,496 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:36,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:39,405 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:40,182 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:40,183 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:40,183 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:40,188 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:40,503 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:40,610 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:00:42,506 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:44,611 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:44,713 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:44,713 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:44,714 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:44,719 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:45,509 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:46,511 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:46,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:00:46,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:00:46,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:00:49,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:49,361 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:49,361 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:49,366 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:49,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:00:50,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:50,516 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:00:55,404 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:00:58,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:00:58,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:00:58,591 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:00:58,596 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:00:59,521 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:00,523 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:00,597 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:01,737 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:01:01,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:01:01,738 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:01:02,864 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:02,865 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:02,865 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:02,871 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:03,525 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:04,528 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:05,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:08,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:08,949 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:08,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:08,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:09,531 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:10,536 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:10,608 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:01:11,609 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:13,366 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:13,367 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:13,367 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:13,372 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:13,538 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:14,541 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:16,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:01:16,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:16,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:01:16,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:01:19,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:19,040 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:19,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:19,045 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:19,545 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:20,548 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:22,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:23,519 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:23,520 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:23,520 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:23,525 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:23,549 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:24,552 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:27,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:27,549 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:27,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:27,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:27,555 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:27,556 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:28,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:31,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:01:31,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:01:31,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:01:33,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:33,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:33,127 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:33,127 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:33,133 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:33,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:34,564 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:38,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:38,440 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:38,441 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:38,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:38,446 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:38,581 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:40,584 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:40,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:01:42,833 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:42,834 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:42,834 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:42,839 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:43,587 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:43,840 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:44,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:46,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:01:46,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:01:46,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:01:47,229 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:47,230 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:47,230 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:47,235 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:47,592 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:48,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:49,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:51,809 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:51,810 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:51,810 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:51,815 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:52,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:52,599 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:54,815 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:01:56,872 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:01:56,873 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:01:56,873 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:01:56,878 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:01:57,601 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:01:58,604 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:01:59,879 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:01,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:02:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:02:01,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:02:01,914 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:02,041 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:02,042 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:02,047 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:02,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:02,608 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:05,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:06,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:06,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:06,331 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:06,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:06,611 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:08,614 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:10,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:10,613 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:02:10,626 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:10,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:10,627 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:10,632 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:11,615 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:12,618 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:15,633 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:16,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:02:16,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:02:16,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:02:18,969 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:18,970 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:18,970 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:18,975 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:19,641 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:20,644 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:21,012 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:23,882 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:23,883 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:23,883 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:23,888 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:24,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:24,648 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:26,889 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:29,438 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:29,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:29,439 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:29,444 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:29,651 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:30,653 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:31,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:02:31,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:02:31,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:02:31,967 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:33,714 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:33,715 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:33,715 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:33,720 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:34,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:34,658 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:37,756 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:40,615 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:02:42,151 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:42,151 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:42,151 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:42,157 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:42,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:42,666 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:43,157 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:46,676 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:46,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:46,677 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:46,683 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:46,738 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:02:46,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:02:46,739 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:02:47,669 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:48,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:48,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:50,550 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:50,550 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:50,550 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:50,556 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:50,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:52,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:54,592 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:02:55,828 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:02:55,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:02:55,829 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:02:55,834 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:02:56,680 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:02:56,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:02:59,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:00,729 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:00,729 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:00,730 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:00,735 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:01,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:03:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:03:01,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:03:02,686 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:05,044 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:06,483 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:06,484 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:06,484 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:06,489 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:06,688 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:08,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:10,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:10,617 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:03:10,803 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:10,804 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:10,804 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:10,809 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:11,693 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:12,696 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:15,468 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:15,469 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:15,469 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:15,474 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:15,698 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:16,474 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:16,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:16,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:03:16,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:03:16,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:03:20,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:20,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:20,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:20,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:20,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:20,706 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:22,184 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:24,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:24,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:24,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:24,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:24,708 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:26,711 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:27,567 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:29,806 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:29,807 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:29,807 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:29,813 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:30,715 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:30,716 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:31,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:03:31,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:03:31,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:03:33,003 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:34,435 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:34,436 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:34,436 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:34,441 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:34,718 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:36,721 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:38,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:38,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:38,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:38,929 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:38,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:39,724 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:40,619 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:03:40,726 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:43,620 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:44,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:44,785 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:44,785 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:44,790 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:45,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:46,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:03:46,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:03:46,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:03:46,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:48,994 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:49,031 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:49,031 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:49,031 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:49,036 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:49,766 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:50,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:54,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:03:54,802 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:54,806 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:54,806 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:54,811 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:55,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:56,774 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:03:58,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:03:58,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:03:58,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:03:58,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:03:59,776 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:03:59,858 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:00,779 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:04:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:04:01,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:04:04,034 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:04,035 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:04,035 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:04,040 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:04,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:04,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:05,076 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:09,429 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:09,430 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:09,430 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:09,435 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:09,786 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:10,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:10,621 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:04:10,788 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:14,355 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:14,356 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:14,356 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:14,361 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:14,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:14,793 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:16,362 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:16,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:04:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:04:16,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:04:20,084 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:20,084 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:20,084 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:20,090 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:20,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:20,800 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:22,128 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:24,508 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:24,509 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:24,509 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:24,514 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:24,802 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:26,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:27,515 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:30,473 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:30,474 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:30,474 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:30,479 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:30,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:04:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:04:31,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:04:32,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:33,030 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:35,330 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:35,331 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:35,331 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:35,337 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:35,814 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:36,816 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:38,372 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:40,500 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:40,501 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:40,501 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:40,506 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:40,623 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:04:40,820 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:42,827 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:43,625 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:46,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:04:46,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:04:46,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:04:48,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:48,740 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:48,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:48,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:48,745 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:48,841 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:50,844 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:53,784 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:04:57,640 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:04:57,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:04:57,641 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:04:57,646 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:04:57,848 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:04:58,851 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:04:59,646 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:01,739 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:05:01,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:05:01,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:05:02,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:02,157 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:02,157 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:02,162 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:02,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:02,855 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:05,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:06,546 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:06,546 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:06,547 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:06,552 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:06,857 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:08,860 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:10,552 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:10,629 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:05:11,450 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:11,451 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:11,451 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:11,456 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:11,920 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:12,923 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:16,457 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:05:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:05:16,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:05:17,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:17,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:17,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:17,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:17,926 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:18,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:21,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:23,888 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:23,889 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:23,889 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:23,894 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:23,932 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:24,934 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:26,895 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:05:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:05:31,740 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:05:31,958 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:32,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:32,252 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:32,252 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:32,257 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:32,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:32,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:37,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:37,917 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:37,917 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:37,918 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:37,928 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:37,975 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:38,978 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:40,627 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:05:42,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:43,177 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:43,178 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:43,178 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:43,183 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:43,981 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:44,985 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:46,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:05:46,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:05:46,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:05:47,496 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:47,497 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:47,497 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:47,503 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:47,987 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:48,540 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:48,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:51,978 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:51,979 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:51,979 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:51,984 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:51,991 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:52,994 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:53,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:05:57,631 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:05:57,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:05:57,632 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:05:57,637 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:05:57,997 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:05:58,999 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:05:59,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:01,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:06:01,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:06:01,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:06:02,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:02,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:02,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:02,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:03,003 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:03,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:05,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:06,863 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:06,864 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:06,864 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:06,869 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:07,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:09,022 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:10,633 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:06:10,634 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:11,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:11,381 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:11,381 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:11,386 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:12,023 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:13,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:15,615 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:15,616 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:15,616 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:15,622 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:16,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:16,622 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:06:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:06:16,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:06:17,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:20,850 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:20,850 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:20,851 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:20,856 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:21,034 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:21,892 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:23,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:25,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:25,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:25,290 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:25,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:26,039 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:27,041 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:27,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:30,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:30,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:30,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:30,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:31,044 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:06:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:06:31,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:06:32,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:33,047 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:36,293 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:36,294 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:36,294 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:36,299 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:37,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:37,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:38,336 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:40,632 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:06:40,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:40,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:40,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:40,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:41,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:43,161 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:43,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:46,419 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:46,420 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:46,420 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:46,425 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:46,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:06:46,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:06:46,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:06:47,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:47,167 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:49,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:50,226 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:50,227 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:50,227 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:50,232 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:51,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:06:51,171 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:55,216 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:06:55,217 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:06:55,217 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:06:55,217 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:06:55,222 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:06:56,174 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:06:57,176 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:00,222 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:00,629 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:00,629 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:00,630 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:00,634 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:01,179 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:01,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:07:01,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:07:01,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:07:03,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:06,055 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:07,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:07,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:07,102 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:07,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:07,184 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:09,188 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:10,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:07:11,638 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:12,095 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:12,095 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:12,096 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:12,101 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:12,189 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:13,192 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:07:16,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:07:16,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:16,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:07:17,269 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:17,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:17,269 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:17,274 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:18,346 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:19,558 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:22,100 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:22,100 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:22,100 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:22,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:22,106 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:22,561 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:23,563 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:27,106 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:30,696 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:30,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:30,697 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:30,702 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:31,568 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:31,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:07:31,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:07:31,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:07:32,997 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:33,571 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:35,190 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:35,190 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:35,191 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:35,197 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:35,572 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:37,576 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:38,232 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:40,635 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:07:42,257 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:42,258 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:42,258 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:42,263 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:42,580 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:43,263 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:43,583 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:46,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:46,269 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:46,269 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:46,275 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:46,585 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:46,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:07:46,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:07:46,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:07:47,588 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:49,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:50,972 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:50,973 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:50,973 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:50,978 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:51,590 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:53,593 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:55,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:07:55,237 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:55,238 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:55,238 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:55,243 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:07:55,594 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:07:57,598 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:07:59,682 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:07:59,683 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:07:59,683 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:07:59,688 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:00,600 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:00,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:01,602 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:01,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:08:01,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:08:01,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:08:05,959 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:08,024 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:08,025 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:08,025 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:08,029 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:08,617 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:09,619 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:10,637 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:08:11,639 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:13,581 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:13,582 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:13,582 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:13,587 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:13,622 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:15,625 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:16,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:08:16,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:16,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:08:16,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:08:19,560 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:19,561 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:19,561 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:19,566 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:19,627 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:21,632 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:22,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:24,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:24,483 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:24,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:24,488 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:24,634 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:25,636 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:28,489 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:29,320 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:29,321 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:29,321 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:29,326 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:29,639 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:31,642 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:31,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:08:31,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:08:31,741 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:08:33,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:36,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:36,698 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:36,698 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:36,703 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:37,645 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:39,672 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:39,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:40,640 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:08:41,164 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:41,164 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:41,165 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:41,169 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:41,673 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:43,676 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:45,170 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:46,036 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:46,037 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:46,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:46,043 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:46,679 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:46,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:08:46,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:08:46,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:08:47,681 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:50,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:50,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:50,686 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:50,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:50,691 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:51,684 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:53,687 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:08:55,728 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:08:56,558 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:08:56,559 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:08:56,559 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:08:56,564 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:08:56,689 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:08:57,691 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:01,565 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:01,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:09:01,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:09:01,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:09:04,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:04,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:04,696 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:04,701 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:05,697 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:06,740 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:07,700 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:09,400 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:09,401 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:09,401 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:09,406 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:09,701 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:10,642 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:09:11,704 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:12,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:15,268 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:15,268 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:15,269 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:15,274 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:15,707 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:16,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:09:16,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:09:16,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:09:17,710 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:18,038 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:20,767 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:20,768 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:20,768 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:20,773 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:21,713 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:23,717 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:23,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:28,812 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:29,118 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:29,119 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:29,119 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:29,124 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:29,720 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:31,723 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:31,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:09:31,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:09:31,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:09:34,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:34,131 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:34,131 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:34,131 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:34,136 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:34,725 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:35,727 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:38,852 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:38,853 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:38,853 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:38,858 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:39,731 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:39,859 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:40,647 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:09:41,762 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:43,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:43,986 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:43,986 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:43,991 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:44,764 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:44,991 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:45,767 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:46,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:09:46,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:09:46,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:09:48,602 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:48,603 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:48,603 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:48,609 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:48,769 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:49,772 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:50,644 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:53,922 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:53,923 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:53,923 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:53,929 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:54,775 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:09:55,777 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:09:55,930 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:09:59,228 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:09:59,228 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:09:59,229 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:09:59,234 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:09:59,780 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:01,234 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:01,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:10:01,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:10:01,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:10:01,783 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:03,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:03,326 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:03,326 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:03,331 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:03,784 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:05,787 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:06,368 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:08,144 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:08,145 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:08,145 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:08,150 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:08,789 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:09,792 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:10,646 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:10:11,647 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:12,875 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:12,876 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:12,876 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:12,881 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:13,794 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:15,797 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:16,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:10:16,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:10:16,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:16,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:10:18,200 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:18,201 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:18,201 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:18,207 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:18,799 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:19,801 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:22,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:23,221 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:23,222 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:23,223 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:23,227 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:23,805 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:25,808 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:27,573 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:27,574 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:27,574 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:27,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:27,579 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:27,809 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:29,812 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:31,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:10:31,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:10:31,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:10:32,937 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:32,938 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:32,938 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:32,938 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:32,943 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:33,815 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:35,818 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:37,980 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:38,672 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:38,673 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:38,673 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:38,678 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:38,821 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:39,823 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:40,651 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:10:43,652 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:46,244 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:46,245 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:46,245 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:46,250 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:46,741 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:10:46,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:10:46,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:10:46,828 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:47,831 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:48,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:53,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:10:54,544 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:10:54,545 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:10:54,545 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:10:54,550 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:10:54,835 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:10:55,837 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:10:59,551 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:01,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:11:01,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:11:01,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:11:03,755 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:03,756 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:03,756 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:03,761 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:03,842 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:04,800 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:05,845 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:08,296 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:08,297 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:08,297 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:08,302 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:08,847 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:09,849 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:10,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:10,650 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:11:12,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:12,139 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:12,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:12,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:12,884 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:13,886 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:16,145 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:16,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:11:16,742 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:11:16,742 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:11:20,443 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:20,444 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:20,444 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:20,449 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:20,891 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:21,488 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:21,893 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:25,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:25,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:25,343 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:25,348 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:25,896 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:27,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:27,899 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:30,310 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:30,311 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:30,311 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:30,316 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:30,900 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:31,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:11:31,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:11:31,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:11:31,903 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:33,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:35,434 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:35,435 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:35,435 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:35,440 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:35,905 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:37,908 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:38,476 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:40,655 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:11:43,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:43,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:43,829 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:43,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:43,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:43,913 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:45,917 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:46,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:11:46,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:11:46,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:11:48,745 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:48,746 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:48,746 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:48,746 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:48,751 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:48,919 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:49,921 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:52,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:52,967 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:52,967 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:52,972 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:53,924 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:53,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:55,927 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:11:58,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:11:58,220 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:11:58,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:11:58,225 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:11:58,928 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:11:59,226 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:11:59,931 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:01,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:12:01,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:12:01,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:12:03,289 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:03,290 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:03,290 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:03,295 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:03,933 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:04,332 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:05,936 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:07,946 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:07,947 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:07,947 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:07,952 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:08,938 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:09,941 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:09,952 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:10,654 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:12:15,656 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:16,180 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:16,181 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:16,181 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:16,186 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:16,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:12:16,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:12:16,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:12:16,954 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:17,957 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:21,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:22,490 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:22,490 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:22,491 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:22,495 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:22,961 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:23,964 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:26,532 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:28,276 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:28,277 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:28,277 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:28,282 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:28,967 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:29,969 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:31,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:12:31,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:12:31,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:12:32,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:36,726 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:36,727 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:36,727 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:36,732 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:36,973 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:37,768 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:37,976 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:40,656 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:12:42,194 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:42,195 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:42,195 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:42,200 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:42,980 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:43,201 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:43,982 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:46,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:12:46,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:12:46,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:12:47,205 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:47,205 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:47,205 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:47,210 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:47,986 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:48,251 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:49,989 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:52,141 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:52,142 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:52,142 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:52,147 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:52,990 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:53,993 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:12:54,148 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:57,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:12:57,354 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:12:57,354 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:12:57,359 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:12:57,995 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:12:59,360 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:12:59,998 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:01,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:13:01,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:13:01,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:13:01,841 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:02,061 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:02,061 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:02,066 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:03,000 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:04,002 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:05,067 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:06,658 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:06,659 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:06,659 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:06,665 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:07,004 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:08,006 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:10,658 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:13:10,660 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:12,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:12,521 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:12,521 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:12,526 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:13,026 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:14,028 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:16,526 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:16,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:13:16,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:13:16,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:13:17,376 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:17,376 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:17,376 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:17,381 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:18,030 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:20,033 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:22,420 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:23,464 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:23,465 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:23,465 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:23,470 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:24,037 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:26,040 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:27,471 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:31,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:13:31,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:13:31,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:13:31,786 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:32,029 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:32,029 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:32,034 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:32,043 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:33,035 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:34,046 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:36,218 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:36,218 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:36,219 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:36,223 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:37,048 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:38,051 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:38,224 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:40,660 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:13:40,942 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:40,943 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:40,943 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:40,949 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:41,054 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:42,056 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:43,949 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:45,855 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:45,856 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:45,856 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:45,861 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:46,060 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:46,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:13:46,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:13:46,769 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:13:48,063 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:48,976 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:50,523 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:50,524 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:50,524 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:50,529 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:51,065 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:52,067 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:13:54,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:13:58,896 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:13:58,896 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:13:58,897 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:13:58,902 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:13:59,071 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:13:59,902 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:00,074 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:01,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:14:01,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:14:01,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:14:04,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:06,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: keepalive +2024-03-13 19:14:07,219 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:07,219 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:07,220 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:07,225 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:08,078 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:09,608 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:09,609 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:09,609 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:09,614 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:10,081 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:10,082 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:10,667 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:14:12,092 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:12,668 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:14,210 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:14,210 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:14,211 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:14,215 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:15,094 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:16,097 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:16,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:14:16,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:14:16,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:14:17,966 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:19,337 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:19,338 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:19,338 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:19,343 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:20,099 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:22,104 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:23,380 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:24,772 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:24,773 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:24,773 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:24,778 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:25,105 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:26,108 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:28,779 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:29,482 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:29,483 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:29,483 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:29,488 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:30,110 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:31,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:14:31,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:14:31,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:14:32,113 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:34,014 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:34,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:34,439 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:34,440 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:34,444 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:35,116 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:36,118 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:39,480 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:39,574 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:39,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:39,575 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:39,580 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:40,122 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:40,665 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:14:42,125 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:44,666 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:46,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:14:46,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:14:46,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:14:47,766 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:47,767 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:47,767 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:47,772 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:48,129 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:49,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:50,132 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:14:54,808 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:14:56,120 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:14:56,121 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:14:56,121 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:14:56,126 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:14:56,136 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:14:58,139 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:00,126 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:00,836 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:00,837 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:00,837 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:00,842 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:01,141 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:01,769 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:15:01,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:15:01,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:15:02,143 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:05,988 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:07,156 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:07,156 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:07,156 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:07,161 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:08,147 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:10,150 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:10,670 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:15:11,575 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:11,575 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:11,576 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:11,577 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:11,582 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:12,151 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:14,154 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:16,328 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:16,329 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:16,329 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:16,334 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:15:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:15:16,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:15:17,156 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:18,158 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:21,416 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:21,417 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:21,417 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:21,422 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:22,162 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:22,460 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:24,165 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:25,829 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:25,830 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:25,830 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:25,835 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:26,166 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:27,835 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:28,169 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:31,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:15:31,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:15:31,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:15:32,984 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:32,985 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:32,985 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:32,985 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:32,990 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:33,172 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:34,175 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:38,016 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:38,016 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:38,017 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:38,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:38,021 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:38,177 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:40,182 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:40,669 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:15:42,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:42,534 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:42,534 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:42,539 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:43,209 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:43,539 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:44,211 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:46,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:15:46,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:15:46,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:15:48,382 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:48,383 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:48,383 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:48,388 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:49,215 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:49,424 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:50,218 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:15:54,139 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:15:54,140 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:15:54,140 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:15:54,145 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:15:54,220 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:15:55,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:15:56,221 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:00,146 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:00,945 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:00,946 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:00,946 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:00,951 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:01,224 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:01,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:16:01,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:16:01,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:16:02,226 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:04,948 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:04,948 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:04,949 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:04,954 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:05,228 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:05,992 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:06,231 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:09,568 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:09,569 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:09,569 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:09,574 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:10,233 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:10,674 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:16:11,675 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:12,236 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:14,764 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:14,765 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:14,765 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:14,770 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:15,278 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:16,281 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:16:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:16:16,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:16:20,352 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:20,353 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:20,353 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:20,358 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:21,285 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:22,288 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:22,396 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:24,670 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:24,671 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:24,671 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:24,676 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:25,289 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:26,292 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:27,677 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:29,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:29,343 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:29,343 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:29,349 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:30,295 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:31,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:16:31,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:16:31,770 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:16:32,298 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:33,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:33,780 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:33,780 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:33,781 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:33,786 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:34,299 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:36,302 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:38,138 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:38,139 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:38,139 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:38,139 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:38,144 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:38,303 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:40,308 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:40,673 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:16:43,455 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:43,456 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:43,456 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:43,456 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:43,461 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:44,310 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:46,342 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:46,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:16:46,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:16:46,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:16:49,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:51,590 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:51,591 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:51,591 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:51,596 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:52,352 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:54,355 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:54,632 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:55,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:55,692 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:55,692 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:55,697 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:16:56,356 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:16:58,360 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:16:59,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:16:59,718 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:16:59,719 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:16:59,719 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:16:59,724 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:00,361 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:01,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:17:01,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:17:01,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:17:02,362 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:05,028 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:05,543 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:05,544 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:05,544 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:05,549 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:06,365 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:08,368 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:10,584 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:10,675 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:17:11,747 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:11,748 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:11,748 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:11,753 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:12,370 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:14,373 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:15,754 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:17:16,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:17:16,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:17:16,781 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:17,065 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:17,066 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:17,071 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:17,392 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:18,395 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:21,071 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:22,342 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:22,342 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:22,343 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:22,347 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:22,398 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:24,401 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:26,349 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:28,009 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:28,010 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:28,010 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:28,015 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:28,404 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:30,407 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:31,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:17:31,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:31,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:17:31,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:17:33,753 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:33,753 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:33,753 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:33,758 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:34,409 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:36,412 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:36,796 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:40,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:40,102 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:40,103 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:40,107 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:40,416 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:40,677 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:17:42,419 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:42,679 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:46,770 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:17:46,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:17:46,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:17:48,072 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:48,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:48,485 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:48,485 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:48,490 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:49,424 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:50,427 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:53,528 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:17:55,358 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:17:55,359 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:17:55,359 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:17:55,364 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:17:55,429 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:17:58,433 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:17:59,365 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:00,029 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:00,030 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:00,030 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:00,035 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:00,435 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:01,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:18:01,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:18:01,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:18:02,438 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:05,019 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:05,890 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:05,891 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:05,891 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:05,896 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:06,440 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:08,444 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:10,679 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:18:10,681 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:10,928 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:10,929 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:10,929 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:10,934 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:11,445 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:12,448 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:15,935 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:16,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:18:16,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:18:16,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:18:17,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:17,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:17,691 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:17,696 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:18,451 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:20,454 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:21,732 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:22,484 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:22,485 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:22,485 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:22,491 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:23,457 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:24,460 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:27,491 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:28,973 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:28,973 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:28,974 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:28,978 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:29,462 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:30,465 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:31,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:18:31,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:18:31,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:18:33,048 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:37,394 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:37,395 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:37,395 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:37,400 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:37,469 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:38,436 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:40,475 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:40,685 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:18:41,907 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:41,907 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:41,908 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:41,912 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:42,499 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:43,913 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:44,502 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:46,520 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:46,521 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:46,521 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:46,526 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:46,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:18:46,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:18:46,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:18:47,504 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:48,507 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:48,977 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:50,427 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:50,428 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:50,428 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:50,433 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:50,508 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:52,511 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:18:54,472 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:18:55,439 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: partial_history +2024-03-13 19:18:55,440 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:18:55,441 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:18:55,445 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:18:55,513 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:18:58,517 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:19:00,446 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:01,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:19:01,771 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:19:01,771 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:19:06,017 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:10,684 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:19:11,685 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:16,686 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:16,903 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:19:16,904 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:19:16,947 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:19:22,110 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:27,111 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:32,101 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:19:32,101 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:19:32,406 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:32,448 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:19:37,407 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:40,686 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:19:42,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:47,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:52,688 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:19:57,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:02,689 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:07,690 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:20:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:20:09,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:20:10,691 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:20:12,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:17,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:22,693 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:20:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:20:24,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:20:28,287 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:33,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:38,288 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:39,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:20:39,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:20:39,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:20:40,690 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:20:43,691 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:48,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:53,692 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:20:54,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:20:54,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:20:54,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:20:59,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:04,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:21:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:21:09,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:21:09,325 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:10,693 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:21:14,694 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:19,695 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:21:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:21:24,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:21:25,353 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:30,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:35,354 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:39,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:21:39,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:21:39,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:21:40,696 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:21:40,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:45,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:50,698 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:21:54,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:21:54,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:21:54,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:21:56,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:01,271 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:06,272 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:22:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:22:09,040 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:22:10,695 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:22:11,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:16,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:21,697 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:22:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:22:24,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:22:27,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:32,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:37,262 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:39,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:22:39,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:22:39,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:22:40,699 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:22:42,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:47,700 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:52,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:22:54,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:22:54,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:22:54,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:22:58,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:03,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:08,285 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:23:09,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:23:09,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:23:10,700 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:23:13,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:18,701 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:23,702 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:23:24,040 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: stop_status +2024-03-13 19:23:24,041 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: stop_status +2024-03-13 19:23:29,261 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:31,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: telemetry +2024-03-13 19:23:31,527 DEBUG SenderThread:4098812 [sender.py:send():382] send: exit +2024-03-13 19:23:31,527 INFO SenderThread:4098812 [sender.py:send_exit():589] handling exit code: 0 +2024-03-13 19:23:31,528 INFO SenderThread:4098812 [sender.py:send_exit():591] handling runtime: 28522 +2024-03-13 19:23:31,533 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:23:31,533 INFO SenderThread:4098812 [sender.py:send_exit():597] send defer +2024-03-13 19:23:31,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:31,533 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-13 19:23:31,533 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:31,533 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-13 19:23:31,533 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 1 +2024-03-13 19:23:31,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:31,533 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-13 19:23:31,533 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:31,533 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-13 19:23:31,533 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 2 +2024-03-13 19:23:31,533 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:31,533 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-13 19:23:31,533 INFO HandlerThread:4098812 [system_monitor.py:finish():203] Stopping system monitor +2024-03-13 19:23:31,534 INFO HandlerThread:4098812 [interfaces.py:finish():202] Joined cpu monitor +2024-03-13 19:23:31,534 INFO HandlerThread:4098812 [interfaces.py:finish():202] Joined disk monitor +2024-03-13 19:23:31,534 DEBUG SystemMonitor:4098812 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-13 19:23:31,535 DEBUG SystemMonitor:4098812 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-13 19:23:32,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:23:32,380 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:23:33,037 INFO HandlerThread:4098812 [interfaces.py:finish():202] Joined gpu monitor +2024-03-13 19:23:33,037 INFO HandlerThread:4098812 [interfaces.py:finish():202] Joined memory monitor +2024-03-13 19:23:33,037 INFO HandlerThread:4098812 [interfaces.py:finish():202] Joined network monitor +2024-03-13 19:23:33,037 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 19:23:33,037 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:33,037 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-13 19:23:33,037 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 3 +2024-03-13 19:23:33,038 DEBUG SenderThread:4098812 [sender.py:send():382] send: stats +2024-03-13 19:23:33,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 19:23:33,039 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:33,039 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-13 19:23:33,039 DEBUG SenderThread:4098812 [sender.py:send():382] send: history +2024-03-13 19:23:33,039 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: summary_record +2024-03-13 19:23:33,044 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:23:33,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:33,045 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-13 19:23:33,045 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 4 +2024-03-13 19:23:33,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:33,045 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-13 19:23:33,045 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:33,045 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-13 19:23:33,045 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 5 +2024-03-13 19:23:33,045 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:33,045 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-13 19:23:33,045 DEBUG SenderThread:4098812 [sender.py:send():382] send: summary +2024-03-13 19:23:33,050 INFO SenderThread:4098812 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-13 19:23:33,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:33,050 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-13 19:23:33,050 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 6 +2024-03-13 19:23:33,050 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:33,050 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-13 19:23:33,050 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:33,050 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-13 19:23:33,053 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: status_report +2024-03-13 19:23:33,303 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 7 +2024-03-13 19:23:33,303 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:33,303 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-13 19:23:33,303 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:33,303 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-13 19:23:33,382 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/config.yaml +2024-03-13 19:23:33,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:23:33,383 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:23:33,503 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 19:23:35,798 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 8 +2024-03-13 19:23:35,798 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 19:23:35,799 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:35,799 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-13 19:23:35,799 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:35,799 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-13 19:23:35,799 INFO SenderThread:4098812 [job_builder.py:build():296] Attempting to build job artifact +2024-03-13 19:23:35,800 INFO SenderThread:4098812 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-13 19:23:35,879 INFO SenderThread:4098812 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-13 19:23:35,908 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 9 +2024-03-13 19:23:35,908 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:35,908 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-13 19:23:35,908 DEBUG SenderThread:4098812 [sender.py:send():382] send: artifact +2024-03-13 19:23:36,384 INFO Thread-12 :4098812 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:23:36,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 19:23:36,944 INFO SenderThread:4098812 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzU0NjY3NjQ2', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzU0NjY3NjQ2', 'versionIndex': 9}}} +2024-03-13 19:23:36,944 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:36,944 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-13 19:23:36,944 INFO SenderThread:4098812 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-13 19:23:37,385 INFO SenderThread:4098812 [dir_watcher.py:finish():388] scan: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/config.yaml config.yaml +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/conda-environment.yaml conda-environment.yaml +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-metadata.json wandb-metadata.json +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/requirements.txt requirements.txt +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log output.log +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [dir_watcher.py:finish():402] scan save: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json wandb-summary.json +2024-03-13 19:23:37,386 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 10 +2024-03-13 19:23:37,387 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 19:23:37,387 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:37,388 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-13 19:23:37,392 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:37,392 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-13 19:23:37,392 INFO SenderThread:4098812 [file_pusher.py:finish():172] shutting down file pusher +2024-03-13 19:23:37,504 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 19:23:37,505 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 19:23:38,140 INFO wandb-upload_2:4098812 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/requirements.txt +2024-03-13 19:23:38,298 INFO wandb-upload_1:4098812 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/conda-environment.yaml +2024-03-13 19:23:38,310 INFO wandb-upload_4:4098812 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/wandb-summary.json +2024-03-13 19:23:38,317 INFO wandb-upload_0:4098812 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/config.yaml +2024-03-13 19:23:38,329 INFO wandb-upload_3:4098812 [upload_job.py:push():131] Uploaded file runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/files/output.log +2024-03-13 19:23:38,505 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 19:23:38,505 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 19:23:38,529 INFO Thread-11 (_thread_body):4098812 [sender.py:transition_state():617] send defer: 11 +2024-03-13 19:23:38,529 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:38,529 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-13 19:23:38,530 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:38,530 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-13 19:23:38,530 INFO SenderThread:4098812 [file_pusher.py:join():178] waiting for file pusher +2024-03-13 19:23:38,530 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 12 +2024-03-13 19:23:38,530 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:38,530 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-13 19:23:38,530 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:38,530 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-13 19:23:38,531 INFO SenderThread:4098812 [file_stream.py:finish():595] file stream finish called +2024-03-13 19:23:39,165 INFO SenderThread:4098812 [file_stream.py:finish():599] file stream finish is done +2024-03-13 19:23:39,165 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 13 +2024-03-13 19:23:39,165 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:39,165 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-13 19:23:39,166 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:39,166 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-13 19:23:39,166 INFO SenderThread:4098812 [sender.py:transition_state():617] send defer: 14 +2024-03-13 19:23:39,166 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: defer +2024-03-13 19:23:39,166 INFO HandlerThread:4098812 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-13 19:23:39,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: final +2024-03-13 19:23:39,166 DEBUG SenderThread:4098812 [sender.py:send():382] send: footer +2024-03-13 19:23:39,166 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: defer +2024-03-13 19:23:39,166 INFO SenderThread:4098812 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-13 19:23:39,167 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-13 19:23:39,167 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: poll_exit +2024-03-13 19:23:39,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-13 19:23:39,168 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: server_info +2024-03-13 19:23:39,168 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: server_info +2024-03-13 19:23:39,211 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: get_summary +2024-03-13 19:23:39,212 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-13 19:23:39,213 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: job_info +2024-03-13 19:23:39,375 DEBUG SenderThread:4098812 [sender.py:send_request():409] send_request: job_info +2024-03-13 19:23:39,375 DEBUG HandlerThread:4098812 [handler.py:handle_request():146] handle_request: shutdown +2024-03-13 19:23:39,375 INFO HandlerThread:4098812 [handler.py:finish():869] shutting down handler +2024-03-13 19:23:40,213 INFO WriterThread:4098812 [datastore.py:close():296] close: runs/s1_s2_qformer2_256/wandb/run-20240313_112808-6hbull8s/run-6hbull8s.wandb +2024-03-13 19:23:40,375 INFO SenderThread:4098812 [sender.py:finish():1572] shutting down sender +2024-03-13 19:23:40,375 INFO SenderThread:4098812 [file_pusher.py:finish():172] shutting down file pusher +2024-03-13 19:23:40,375 INFO SenderThread:4098812 [file_pusher.py:join():178] waiting for file pusher